From 202a1ab635aca5d999738fdcd7512618f4d3e010 Mon Sep 17 00:00:00 2001 From: RishiDiwanTT <90382027+RishiDiwanTT@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:55:40 +0530 Subject: [PATCH] PP-149 Refactor Acquisition feeds and Annotators (#1308) * OPDS 1.2 and 2.0 refactor With an acquisition page feed This aims to remove all XML directives from within the feed and annotator workflows And also to simplify the workflow making it more linear and less circular Only a Serializer should write XML to the feed Serializers are picked up the the basis of the Accept header sent to the endpoint. Currently only the /feed, /group and /loans endpoints react to the Accept headers, the rest are still OPDS 1.2 --- api/admin/controller/custom_lists.py | 12 +- api/admin/controller/work_editor.py | 8 +- api/annotations.py | 2 +- api/circulation.py | 5 - api/controller.py | 119 +- core/app_server.py | 13 +- core/feed/acquisition.py | 926 +++++++++ core/feed/admin.py | 70 + core/feed/annotator/admin.py | 105 + core/feed/annotator/base.py | 389 ++++ core/feed/annotator/circulation.py | 1564 ++++++++++++++ core/feed/annotator/loan_and_hold.py | 125 ++ core/feed/annotator/verbose.py | 110 + core/feed/base.py | 13 + core/feed/navigation.py | 92 + core/feed/opds.py | 96 + core/feed/serializer/base.py | 32 + core/feed/serializer/opds.py | 363 ++++ core/feed/serializer/opds2.py | 213 ++ core/feed/types.py | 240 +++ core/feed/util.py | 27 + core/model/constants.py | 1 + core/model/licensing.py | 6 +- core/model/work.py | 4 +- core/opds_schema.py | 2 + pyproject.toml | 1 + scripts.py | 17 +- .../feed/equivalence/test_feed_equivalence.py | 296 +++ tests/api/feed/fixtures.py | 47 + tests/api/feed/test_admin.py | 285 +++ tests/api/feed/test_annotators.py | 469 +++++ tests/api/feed/test_library_annotator.py | 1795 +++++++++++++++++ .../api/feed/test_loan_and_hold_annotator.py | 287 +++ tests/api/feed/test_opds2_serializer.py | 215 ++ tests/api/feed/test_opds_acquisition_feed.py | 1454 +++++++++++++ tests/api/feed/test_opds_base.py | 57 + tests/api/feed/test_opds_serializer.py | 232 +++ tests/api/test_controller_cm.py | 5 +- tests/api/test_controller_crawlfeed.py | 15 +- tests/api/test_controller_loan.py | 2 +- tests/api/test_controller_opdsfeed.py | 37 +- tests/api/test_controller_work.py | 64 +- tests/api/test_scripts.py | 12 +- tests/core/test_app_server.py | 6 +- 44 files changed, 9684 insertions(+), 149 deletions(-) create mode 100644 core/feed/acquisition.py create mode 100644 core/feed/admin.py create mode 100644 core/feed/annotator/admin.py create mode 100644 core/feed/annotator/base.py create mode 100644 core/feed/annotator/circulation.py create mode 100644 core/feed/annotator/loan_and_hold.py create mode 100644 core/feed/annotator/verbose.py create mode 100644 core/feed/base.py create mode 100644 core/feed/navigation.py create mode 100644 core/feed/opds.py create mode 100644 core/feed/serializer/base.py create mode 100644 core/feed/serializer/opds.py create mode 100644 core/feed/serializer/opds2.py create mode 100644 core/feed/types.py create mode 100644 core/feed/util.py create mode 100644 tests/api/feed/equivalence/test_feed_equivalence.py create mode 100644 tests/api/feed/fixtures.py create mode 100644 tests/api/feed/test_admin.py create mode 100644 tests/api/feed/test_annotators.py create mode 100644 tests/api/feed/test_library_annotator.py create mode 100644 tests/api/feed/test_loan_and_hold_annotator.py create mode 100644 tests/api/feed/test_opds2_serializer.py create mode 100644 tests/api/feed/test_opds_acquisition_feed.py create mode 100644 tests/api/feed/test_opds_base.py create mode 100644 tests/api/feed/test_opds_serializer.py diff --git a/api/admin/controller/custom_lists.py b/api/admin/controller/custom_lists.py index b2e72c10cf..c117aa215c 100644 --- a/api/admin/controller/custom_lists.py +++ b/api/admin/controller/custom_lists.py @@ -24,6 +24,7 @@ from api.controller import CirculationManagerController from api.problem_details import CANNOT_DELETE_SHARED_LIST from core.app_server import load_pagination_from_request +from core.feed.acquisition import OPDSAcquisitionFeed from core.lane import Lane, WorkList from core.model import ( Collection, @@ -36,10 +37,8 @@ create, get_one, ) -from core.opds import AcquisitionFeed from core.problem_details import INVALID_INPUT, METHOD_NOT_ALLOWED from core.query.customlist import CustomListQueries -from core.util.flask_util import OPDSFeedResponse from core.util.problem_detail import ProblemDetail @@ -351,12 +350,11 @@ def custom_list( annotator = self.manager.annotator(worklist) url_fn = self.url_for_custom_list(library, list) - feed = AcquisitionFeed.from_query( - query, self._db, list.name, url, pagination, url_fn, annotator + feed = OPDSAcquisitionFeed.from_query( + query, self._db, list.name or "", url, pagination, url_fn, annotator ) - annotator.annotate_feed(feed, worklist) - - return OPDSFeedResponse(str(feed), max_age=0) + annotator.annotate_feed(feed) + return feed.as_response(max_age=0) elif flask.request.method == "POST": ctx: Context = flask.request.context.body # type: ignore diff --git a/api/admin/controller/work_editor.py b/api/admin/controller/work_editor.py index 6b8f520f31..b8c41014fb 100644 --- a/api/admin/controller/work_editor.py +++ b/api/admin/controller/work_editor.py @@ -12,10 +12,11 @@ from flask_babel import lazy_gettext as _ from PIL import Image, ImageDraw, ImageFont -from api.admin.opds import AdminAnnotator from api.admin.problem_details import * from api.admin.validator import Validator from core.classifier import NO_NUMBER, NO_VALUE, SimplifiedGenreClassifier, genres +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.admin import AdminAnnotator from core.lane import Lane from core.metadata_layer import LinkData, Metadata, ReplacementPolicy from core.mirror import MirrorUploader @@ -37,7 +38,6 @@ get_one_or_create, ) from core.model.configuration import ExternalIntegrationLink -from core.opds import AcquisitionFeed from core.util import LanguageCodes from core.util.datetime_helpers import strptime_utc, utc_now from core.util.problem_detail import ProblemDetail @@ -68,7 +68,9 @@ def details(self, identifier_type, identifier): # single_entry returns an OPDSEntryResponse that will not be # cached, which is perfect. We want the admin interface # to update immediately when an admin makes a change. - return AcquisitionFeed.single_entry(self._db, work, annotator) + return OPDSAcquisitionFeed.entry_as_response( + OPDSAcquisitionFeed.single_entry(work, annotator) + ) def roles(self): """Return a mapping from MARC codes to contributor roles.""" diff --git a/api/annotations.py b/api/annotations.py index 2fa5cf6360..d2de031610 100644 --- a/api/annotations.py +++ b/api/annotations.py @@ -1,9 +1,9 @@ import json import os +from flask import url_for from pyld import jsonld -from core.app_server import url_for from core.model import Annotation, Identifier from core.util.datetime_helpers import utc_now diff --git a/api/circulation.py b/api/circulation.py index 078bea0613..9765b0801e 100644 --- a/api/circulation.py +++ b/api/circulation.py @@ -1354,11 +1354,6 @@ def enforce_limits(self, patron: Patron, pool: LicensePool) -> None: if api is not None: api.update_availability(pool) - if pool.licenses_available is None: - # We don't know how many licenses are available, so we - # can't tell whether the patron is at their limit. - self.log.warning(f"License pool {pool} has unknown availability.") - return currently_available = pool.licenses_available > 0 if currently_available and at_loan_limit: raise PatronLoanLimitReached(limit=patron.library.settings.loan_limit) diff --git a/api/controller.py b/api/controller.py index 743697e886..6b07889caf 100644 --- a/api/controller.py +++ b/api/controller.py @@ -21,11 +21,12 @@ from sqlalchemy import select from sqlalchemy.orm import eagerload from sqlalchemy.orm.exc import NoResultFound +from werkzeug.datastructures import MIMEAccept from api.authentication.access_token import AccessTokenProvider from api.model.patron_auth import PatronAuthAccessToken from api.model.time_tracking import PlaytimeEntriesPost, PlaytimeEntriesPostResponse -from api.opds2 import OPDS2NavigationsAnnotator, OPDS2PublicationsAnnotator +from api.opds2 import OPDS2NavigationsAnnotator from api.saml.controller import SAMLController from core.analytics import Analytics from core.app_server import ApplicationVersionController @@ -37,6 +38,12 @@ ) from core.entrypoint import EverythingEntryPoint from core.external_search import ExternalSearchIndex, SortKeyPagination +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.circulation import ( + CirculationManagerAnnotator, + LibraryAnnotator, +) +from core.feed.navigation import NavigationFeed from core.lane import ( BaseFacets, Facets, @@ -76,7 +83,7 @@ InvalidTokenTypeError, ) from core.model.discovery_service_registration import DiscoveryServiceRegistration -from core.opds import AcquisitionFeed, NavigationFacets, NavigationFeed +from core.opds import NavigationFacets from core.opds2 import AcquisitonFeedOPDS2 from core.opensearch import OpenSearchDocument from core.query.playtime_entries import PlaytimeEntries @@ -112,11 +119,6 @@ ) from .odl import ODLAPI from .odl2 import ODL2API -from .opds import ( - CirculationManagerAnnotator, - LibraryAnnotator, - LibraryLoanAndHoldAnnotator, -) from .problem_details import * if TYPE_CHECKING: @@ -842,7 +844,7 @@ def appropriate_index_for_patron_type(self): class OPDSFeedController(CirculationManagerController): - def groups(self, lane_identifier, feed_class=AcquisitionFeed): + def groups(self, lane_identifier, feed_class=OPDSAcquisitionFeed): """Build or retrieve a grouped acquisition feed. :param lane_identifier: An identifier that uniquely identifiers @@ -910,9 +912,9 @@ def groups(self, lane_identifier, feed_class=AcquisitionFeed): annotator=annotator, facets=facets, search_engine=search_engine, - ) + ).as_response(mime_types=flask.request.accept_mimetypes) - def feed(self, lane_identifier, feed_class=AcquisitionFeed): + def feed(self, lane_identifier, feed_class=OPDSAcquisitionFeed): """Build or retrieve a paginated acquisition feed. :param lane_identifier: An identifier that uniquely identifiers @@ -943,7 +945,7 @@ def feed(self, lane_identifier, feed_class=AcquisitionFeed): annotator = self.manager.annotator(lane, facets=facets) max_age = flask.request.args.get("max_age") - return feed_class.page( + feed = feed_class.page( _db=self._db, title=lane.display_name, url=url, @@ -952,7 +954,10 @@ def feed(self, lane_identifier, feed_class=AcquisitionFeed): facets=facets, pagination=pagination, search_engine=search_engine, + ) + return feed.as_response( max_age=int(max_age) if max_age else None, + mime_types=flask.request.accept_mimetypes, ) def navigation(self, lane_identifier): @@ -987,7 +992,7 @@ def navigation(self, lane_identifier): worklist=lane, annotator=annotator, facets=facets, - ) + ).as_response() def crawlable_library_feed(self): """Build or retrieve a crawlable acquisition feed for the @@ -1043,7 +1048,7 @@ def crawlable_list_feed(self, list_name): return self._crawlable_feed(title=title, url=url, worklist=lane) def _crawlable_feed( - self, title, url, worklist, annotator=None, feed_class=AcquisitionFeed + self, title, url, worklist, annotator=None, feed_class=OPDSAcquisitionFeed ): """Helper method to create a crawlable feed. @@ -1052,7 +1057,7 @@ def _crawlable_feed( :param worklist: A crawlable Lane which controls which works show up in the feed. :param annotator: A custom Annotator to use when generating the feed. - :param feed_class: A drop-in replacement for AcquisitionFeed + :param feed_class: A drop-in replacement for OPDSAcquisitionFeed for use in tests. """ pagination = load_pagination_from_request( @@ -1080,7 +1085,7 @@ def _crawlable_feed( facets=facets, pagination=pagination, search_engine=search_engine, - ) + ).as_response() def _load_search_facets(self, lane): entrypoints = list(flask.request.library.entrypoints) @@ -1098,7 +1103,7 @@ def _load_search_facets(self, lane): default_entrypoint=default_entrypoint, ) - def search(self, lane_identifier, feed_class=AcquisitionFeed): + def search(self, lane_identifier, feed_class=OPDSAcquisitionFeed): """Search for books.""" lane = self.load_lane(lane_identifier) if isinstance(lane, ProblemDetail): @@ -1153,7 +1158,7 @@ def search(self, lane_identifier, feed_class=AcquisitionFeed): # Run a search. annotator = self.manager.annotator(lane, facets) info = OpenSearchDocument.search_info(lane) - return feed_class.search( + response = feed_class.search( _db=self._db, title=info["name"], url=make_url(), @@ -1164,6 +1169,9 @@ def search(self, lane_identifier, feed_class=AcquisitionFeed): pagination=pagination, facets=facets, ) + if isinstance(response, ProblemDetail): + return response + return response.as_response(mime_types=flask.request.accept_mimetypes) def _qa_feed( self, feed_factory, feed_title, controller_name, facet_class, worklist_factory @@ -1218,7 +1226,7 @@ def _qa_feed( max_age=CachedFeed.IGNORE_CACHE, ) - def qa_feed(self, feed_class=AcquisitionFeed): + def qa_feed(self, feed_class=OPDSAcquisitionFeed): """Create an OPDS feed containing the information necessary to run a full set of integration tests against this server and the vendors it relies on. @@ -1238,7 +1246,7 @@ def factory(library, facets): worklist_factory=factory, ) - def qa_series_feed(self, feed_class=AcquisitionFeed): + def qa_series_feed(self, feed_class=OPDSAcquisitionFeed): """Create an OPDS feed containing books that belong to _some_ series, without regard to _which_ series. @@ -1297,23 +1305,22 @@ def publications(self): params: FeedRequestParameters = self._parse_feed_request() if params.problem: return params.problem - annotator = OPDS2PublicationsAnnotator( - flask.request.url, params.facets, params.pagination, params.library - ) lane = self.load_lane(None) + annotator = self.manager.annotator(lane, params.facets) max_age = flask.request.args.get("max_age") - feed = AcquisitonFeedOPDS2.publications( + feed = OPDSAcquisitionFeed.page( self._db, + lane.display_name, + flask.request.url, lane, + annotator, params.facets, params.pagination, self.search_engine, - annotator, - max_age=int(max_age) if max_age is not None else None, ) - - return Response( - str(feed), status=200, headers={"Content-Type": annotator.OPDS2_TYPE} + return feed.as_response( + mime_types=MIMEAccept([("application/opds+json", 1)]), # Force the type + max_age=int(max_age) if max_age is not None else None, ) def navigation(self): @@ -1456,7 +1463,17 @@ def sync(self): ) # Then make the feed. - return LibraryLoanAndHoldAnnotator.active_loans_for(self.circulation, patron) + feed = OPDSAcquisitionFeed.active_loans_for(self.circulation, patron) + response = feed.as_response( + max_age=0, + private=True, + mime_types=flask.request.accept_mimetypes, + ) + + last_modified = patron.last_loan_activity_sync + if last_modified: + response.last_modified = last_modified + return response def borrow(self, identifier_type, identifier, mechanism_id=None): """Create a new loan or hold for a book. @@ -1504,7 +1521,7 @@ def borrow(self, identifier_type, identifier, mechanism_id=None): response_kwargs["status"] = 201 else: response_kwargs["status"] = 200 - return LibraryLoanAndHoldAnnotator.single_item_feed( + return OPDSAcquisitionFeed.single_entry_loans_feed( self.circulation, loan_or_hold, **response_kwargs ) @@ -1790,7 +1807,7 @@ def fulfill_part_url(part): if mechanism.delivery_mechanism.is_streaming: # If this is a streaming delivery mechanism, create an OPDS entry # with a fulfillment link to the streaming reader url. - feed = LibraryLoanAndHoldAnnotator.single_item_feed( + feed = OPDSAcquisitionFeed.single_entry_loans_feed( self.circulation, loan, fulfillment=fulfillment ) if isinstance(feed, ProblemDetail): @@ -1799,8 +1816,6 @@ def fulfill_part_url(part): return feed if isinstance(feed, Response): return feed - if isinstance(feed, OPDSFeed): # type: ignore - content = str(feed) else: content = etree.tostring(feed) status_code = 200 @@ -1917,7 +1932,9 @@ def revoke(self, license_pool_id): work = pool.work annotator = self.manager.annotator(None) - return AcquisitionFeed.single_entry(self._db, work, annotator) + return OPDSAcquisitionFeed.entry_as_response( + OPDSAcquisitionFeed.single_entry(work, annotator) + ) def detail(self, identifier_type, identifier): if flask.request.method == "DELETE": @@ -1949,7 +1966,7 @@ def detail(self, identifier_type, identifier): item = loan else: item = hold - return LibraryLoanAndHoldAnnotator.single_item_feed(self.circulation, item) + return OPDSAcquisitionFeed.single_entry_loans_feed(self.circulation, item) class AnnotationController(CirculationManagerController): @@ -2042,7 +2059,7 @@ def _lane_details(self, languages, audiences): return languages, audiences def contributor( - self, contributor_name, languages, audiences, feed_class=AcquisitionFeed + self, contributor_name, languages, audiences, feed_class=OPDSAcquisitionFeed ): """Serve a feed of books written by a particular author""" library = flask.request.library @@ -2096,7 +2113,7 @@ def contributor( pagination=pagination, annotator=annotator, search_engine=search_engine, - ) + ).as_response() def permalink(self, identifier_type, identifier): """Serve an entry for a single book. @@ -2129,18 +2146,23 @@ def permalink(self, identifier_type, identifier): item = loan or hold pool = pool or pools[0] - return LibraryLoanAndHoldAnnotator.single_item_feed( + return OPDSAcquisitionFeed.single_entry_loans_feed( self.circulation, item or pool ) else: annotator = self.manager.annotator(lane=None) - return AcquisitionFeed.single_entry( - self._db, work, annotator, max_age=OPDSFeed.DEFAULT_MAX_AGE + return OPDSAcquisitionFeed.entry_as_response( + OPDSAcquisitionFeed.single_entry(work, annotator), + max_age=OPDSFeed.DEFAULT_MAX_AGE, ) def related( - self, identifier_type, identifier, novelist_api=None, feed_class=AcquisitionFeed + self, + identifier_type, + identifier, + novelist_api=None, + feed_class=OPDSAcquisitionFeed, ): """Serve a groups feed of books related to a given book.""" @@ -2185,12 +2207,17 @@ def related( url=url, worklist=lane, annotator=annotator, + pagination=None, facets=facets, search_engine=search_engine, - ) + ).as_response() def recommendations( - self, identifier_type, identifier, novelist_api=None, feed_class=AcquisitionFeed + self, + identifier_type, + identifier, + novelist_api=None, + feed_class=OPDSAcquisitionFeed, ): """Serve a feed of recommendations related to a given book.""" @@ -2242,9 +2269,9 @@ def recommendations( pagination=pagination, annotator=annotator, search_engine=search_engine, - ) + ).as_response() - def series(self, series_name, languages, audiences, feed_class=AcquisitionFeed): + def series(self, series_name, languages, audiences, feed_class=OPDSAcquisitionFeed): """Serve a feed of books in a given series.""" library = flask.request.library if not series_name: @@ -2281,7 +2308,7 @@ def series(self, series_name, languages, audiences, feed_class=AcquisitionFeed): pagination=pagination, annotator=annotator, search_engine=search_engine, - ) + ).as_response() class ProfileController(CirculationManagerController): diff --git a/core/app_server.py b/core/app_server.py index 96f265fe90..5eaa9464b7 100644 --- a/core/app_server.py +++ b/core/app_server.py @@ -16,13 +16,12 @@ import core from api.admin.config import Configuration as AdminUiConfig +from core.feed.acquisition import LookupAcquisitionFeed, OPDSAcquisitionFeed from .lane import Facets, Pagination from .log import LogConfiguration from .model import Identifier -from .opds import AcquisitionFeed, LookupAcquisitionFeed from .problem_details import * -from .util.flask_util import OPDSFeedResponse from .util.opds_writer import OPDSMessage from .util.problem_detail import ProblemDetail @@ -303,16 +302,15 @@ def work_lookup(self, annotator, route_name="lookup", **process_urn_kwargs): if isinstance(handler, ProblemDetail): # In a subclass, self.process_urns may return a ProblemDetail return handler - opds_feed = LookupAcquisitionFeed( - self._db, "Lookup results", this_url, handler.works, annotator, precomposed_entries=handler.precomposed_entries, ) - return OPDSFeedResponse(str(opds_feed)) + opds_feed.generate_feed(annotate=False) + return opds_feed.as_response() def process_urns(self, urns, **process_urn_kwargs): """Process a number of URNs by instantiating a URNLookupHandler @@ -342,15 +340,14 @@ def permalink(self, urn, annotator, route_name="work"): # work) tuples, but an AcquisitionFeed's .works is just a # list of works. works = [work for (identifier, work) in handler.works] - opds_feed = AcquisitionFeed( - self._db, + opds_feed = OPDSAcquisitionFeed( urn, this_url, works, annotator, precomposed_entries=handler.precomposed_entries, ) - return OPDSFeedResponse(str(opds_feed)) + return opds_feed.as_response() class URNLookupHandler: diff --git a/core/feed/acquisition.py b/core/feed/acquisition.py new file mode 100644 index 0000000000..12713e1daf --- /dev/null +++ b/core/feed/acquisition.py @@ -0,0 +1,926 @@ +"""OPDS feeds, they can be serialized to either OPDS 1 or OPDS 2""" +from __future__ import annotations + +import logging +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + List, + Optional, + Tuple, + Type, +) + +from sqlalchemy.orm import Query, Session + +from api.problem_details import NOT_FOUND_ON_REMOTE +from core.entrypoint import EntryPoint +from core.external_search import ExternalSearchIndex, QueryParseException +from core.facets import FacetConstants +from core.feed.annotator.base import Annotator +from core.feed.annotator.circulation import ( + CirculationManagerAnnotator, + LibraryAnnotator, +) +from core.feed.annotator.loan_and_hold import LibraryLoanAndHoldAnnotator +from core.feed.opds import BaseOPDSFeed +from core.feed.types import FeedData, Link, WorkEntry +from core.feed.util import strftime +from core.lane import Facets, FacetsWithEntryPoint, Lane, Pagination, SearchFacets +from core.model.constants import LinkRelations +from core.model.edition import Edition +from core.model.identifier import Identifier +from core.model.licensing import LicensePool +from core.model.patron import Hold, Loan, Patron +from core.model.work import Work +from core.opds import UnfulfillableWork +from core.problem_details import INVALID_INPUT +from core.util.datetime_helpers import utc_now +from core.util.flask_util import OPDSEntryResponse, OPDSFeedResponse +from core.util.opds_writer import OPDSMessage +from core.util.problem_detail import ProblemDetail + +if TYPE_CHECKING: + from api.circulation import CirculationAPI, FulfillmentInfo + from core.lane import WorkList + + +class OPDSAcquisitionFeed(BaseOPDSFeed): + """An Acquisition Feed which is not tied to any particular format. + It is simply responsible for creating different types of feeds.""" + + def __init__( + self, + title: str, + url: str, + works: List[Work], + annotator: CirculationManagerAnnotator, + facets: Optional[FacetsWithEntryPoint] = None, + pagination: Optional[Pagination] = None, + precomposed_entries: Optional[List[OPDSMessage]] = None, + ) -> None: + self.annotator = annotator + self._facets = facets + self._pagination = pagination + super().__init__(title, url, precomposed_entries=precomposed_entries) + for work in works: + entry = self.single_entry(work, self.annotator) + if isinstance(entry, WorkEntry): + self._feed.entries.append(entry) + + def generate_feed(self, annotate: bool = True) -> None: + """Generate the feed metadata and links. + We assume the entries have already been annotated.""" + self._feed.add_metadata("id", text=self.url) + self._feed.add_metadata("title", text=self.title) + self._feed.add_metadata("updated", text=strftime(utc_now())) + self._feed.add_link(href=self.url, rel="self") + if annotate: + self.annotator.annotate_feed(self._feed) + + def add_pagination_links(self, works: List[Work], lane: WorkList) -> None: + """Add pagination links to the feed""" + if not self._pagination: + return None + if len(works) and self._pagination.has_next_page: + self._feed.add_link( + href=self.annotator.feed_url( + lane, self._facets, self._pagination.next_page + ), + rel="next", + ) + + if self._pagination.offset > 0: + self._feed.add_link( + href=self.annotator.feed_url( + lane, self._facets, self._pagination.first_page + ), + rel="first", + ) + + if self._pagination.previous_page: + self._feed.add_link( + href=self.annotator.feed_url( + lane, self._facets, self._pagination.previous_page + ), + rel="previous", + ) + + def add_facet_links(self, lane: WorkList) -> None: + """Add facet links to the feed""" + if self._facets is None: + return None + else: + facets = self._facets + entrypoints = facets.selectable_entrypoints(lane) + if entrypoints: + # A paginated feed may have multiple entry points into the + # same dataset. + def make_link(ep: Type[EntryPoint]) -> str: + return self.annotator.feed_url( + lane, facets=facets.navigate(entrypoint=ep) + ) + + self.add_entrypoint_links( + self._feed, make_link, entrypoints, facets.entrypoint + ) + + # Facet links + facet_links = self.facet_links(self.annotator, self._facets) + for linkdata in facet_links: + self._feed.facet_links.append(linkdata) + + @classmethod + def facet_links( + cls, annotator: CirculationManagerAnnotator, facets: FacetsWithEntryPoint + ) -> Generator[Link, None, None]: + """Create links for this feed's navigational facet groups. + + This does not create links for the entry point facet group, + because those links should only be present in certain + circumstances, and this method doesn't know if those + circumstances apply. You need to decide whether to call + add_entrypoint_links in addition to calling this method. + """ + for group, value, new_facets, selected in facets.facet_groups: + url = annotator.facet_url(new_facets) + if not url: + continue + group_title = Facets.GROUP_DISPLAY_TITLES.get(group) + facet_title = Facets.FACET_DISPLAY_TITLES.get(value) + if not facet_title: + display_lambda = Facets.FACET_DISPLAY_TITLES_DYNAMIC.get(group) + facet_title = display_lambda(new_facets) if display_lambda else None + if not (group_title and facet_title): + # This facet group or facet, is not recognized by the + # system. It may be left over from an earlier version, + # or just weird junk data. + continue + yield cls.facet_link(url, str(facet_title), str(group_title), selected) + + @classmethod + def facet_link( + cls, href: str, title: str, facet_group_name: str, is_active: bool + ) -> Link: + """Build a set of attributes for a facet link. + + :param href: Destination of the link. + :param title: Human-readable description of the facet. + :param facet_group_name: The facet group to which the facet belongs, + e.g. "Sort By". + :param is_active: True if this is the client's currently + selected facet. + + :retusrn: A dictionary of attributes, suitable for passing as + keyword arguments into OPDSFeed.add_link_to_feed. + """ + args = dict(href=href, title=title) + args["rel"] = LinkRelations.FACET_REL + args["facetGroup"] = facet_group_name + if is_active: + args["activeFacet"] = "true" + return Link.create(**args) + + def as_error_response(self, **kwargs: Any) -> OPDSFeedResponse: + """Convert this feed into an OPDSFeedResponse that should be treated + by intermediaries as an error -- that is, treated as private + and not cached. + """ + kwargs["max_age"] = 0 + kwargs["private"] = True + return self.as_response(**kwargs) + + @classmethod + def _create_entry( + cls, + work: Work, + active_licensepool: Optional[LicensePool], + edition: Edition, + identifier: Identifier, + annotator: Annotator, + ) -> WorkEntry: + entry = WorkEntry( + work=work, + edition=edition, + identifier=identifier, + license_pool=active_licensepool, + ) + annotator.annotate_work_entry(entry) + return entry + + ## OPDS1 specifics + @classmethod + def add_entrypoint_links( + cls, + feed: FeedData, + url_generator: Callable[[Type[EntryPoint]], str], + entrypoints: List[Type[EntryPoint]], + selected_entrypoint: Optional[Type[EntryPoint]], + group_name: str = "Formats", + ) -> None: + """Add links to a feed forming an OPDS facet group for a set of + EntryPoints. + + :param feed: A FeedData object. + :param url_generator: A callable that returns the entry point + URL when passed an EntryPoint. + :param entrypoints: A list of all EntryPoints in the facet group. + :param selected_entrypoint: The current EntryPoint, if selected. + """ + if len(entrypoints) == 1 and selected_entrypoint in (None, entrypoints[0]): + # There is only one entry point. Unless the currently + # selected entry point is somehow different, there's no + # need to put any links at all here -- a facet group with + # one one facet might as well not be there. + return + + is_default = True + for entrypoint in entrypoints: + link = cls._entrypoint_link( + url_generator, entrypoint, selected_entrypoint, is_default, group_name + ) + if link is not None: + feed.links.append(link) + is_default = False + + @classmethod + def _entrypoint_link( + cls, + url_generator: Callable[[Type[EntryPoint]], str], + entrypoint: Type[EntryPoint], + selected_entrypoint: Optional[Type[EntryPoint]], + is_default: bool, + group_name: str, + ) -> Optional[Link]: + """Create arguments for add_link_to_feed for a link that navigates + between EntryPoints. + """ + display_title = EntryPoint.DISPLAY_TITLES.get(entrypoint) + if not display_title: + # Shouldn't happen. + return None + + url = url_generator(entrypoint) + is_selected = entrypoint is selected_entrypoint + link = cls.facet_link(url, display_title, group_name, is_selected) + + # Unlike a normal facet group, every link in this facet + # group has an additional attribute marking it as an entry + # point. + # + # In OPDS 2 this can become an additional rel value, + # removing the need for a custom attribute. + link.add_attributes({"facetGroupType": FacetConstants.ENTRY_POINT_REL}) + return link + + def add_breadcrumb_links( + self, lane: WorkList, entrypoint: Optional[Type[EntryPoint]] = None + ) -> None: + """Add information necessary to find your current place in the + site's navigation. + + A link with rel="start" points to the start of the site + + An Entrypoint section describes the current entry point. + + A breadcrumbs section contains a sequence of breadcrumb links. + """ + # Add the top-level link with rel='start' + annotator = self.annotator + top_level_title = annotator.top_level_title() or "Collection Home" + self.add_link(annotator.default_lane_url(), rel="start", title=top_level_title) + + # Add a link to the direct parent with rel="up". + # + # TODO: the 'direct parent' may be the same lane but without + # the entry point specified. Fixing this would also be a good + # opportunity to refactor the code for figuring out parent and + # parent_title. + parent = None + if isinstance(lane, Lane): + parent = lane.parent + if parent and parent.display_name: + parent_title = parent.display_name + else: + parent_title = top_level_title + + if parent: + up_uri = annotator.lane_url(parent) + self.add_link(up_uri, rel="up", title=parent_title) + self.add_breadcrumbs(lane, entrypoint=entrypoint) + + # Annotate the feed with a simplified:entryPoint for the + # current EntryPoint. + self.show_current_entrypoint(entrypoint) + + def add_breadcrumbs( + self, + lane: WorkList, + include_lane: bool = False, + entrypoint: Optional[Type[EntryPoint]] = None, + ) -> None: + """Add list of ancestor links in a breadcrumbs element. + + :param lane: Add breadcrumbs from up to this lane. + :param include_lane: Include `lane` itself in the breadcrumbs. + :param entrypoint: The currently selected entrypoint, if any. + + TODO: The switchover from "no entry point" to "entry point" needs + its own breadcrumb link. + """ + if entrypoint is None: + entrypoint_query = "" + else: + entrypoint_query = "?entrypoint=" + entrypoint.INTERNAL_NAME + + # Breadcrumbs for lanes may be end up being cut off by a + # patron-type-specific root lane. If so, that lane -- not the + # site root -- should become the first breadcrumb. + site_root_lane = None + usable_parentage = [] + if lane is not None: + for ancestor in [lane] + list(lane.parentage): + if isinstance(ancestor, Lane) and ancestor.root_for_patron_type: + # Root lane for a specific patron type. The root is + # treated specially, so it should not be added to + # usable_parentage. Any lanes between this lane and the + # library root should not be included at all. + site_root_lane = ancestor + break + + if ancestor != lane or include_lane: + # A lane may appear in its own breadcrumbs + # only if include_lane is True. + usable_parentage.append(ancestor) + + annotator = self.annotator + if lane == site_root_lane or ( + site_root_lane is None + and annotator.lane_url(lane) == annotator.default_lane_url() + ): + # There are no extra breadcrumbs: either we are at the + # site root, or we are at a lane that is the root for a + # specific patron type. + return + + breadcrumbs = [] + + # Add root link. This is either the link to the site root + # or to the root lane for some patron type. + if site_root_lane is None: + root_url = annotator.default_lane_url() + root_title = annotator.top_level_title() + else: + root_url = annotator.lane_url(site_root_lane) + root_title = site_root_lane.display_name + root_link = Link(href=root_url, title=root_title) + breadcrumbs.append(root_link) + + # Add entrypoint selection link + if entrypoint: + breadcrumbs.append( + Link( + href=root_url + entrypoint_query, + title=entrypoint.INTERNAL_NAME, + ) + ) + + # Add links for all usable lanes between `lane` and `site_root_lane` + # (possibly including `lane` itself). + for ancestor in reversed(usable_parentage): + lane_url = annotator.lane_url(ancestor) + if lane_url == root_url: + # Root lane for the entire site. + break + + breadcrumbs.append( + Link( + href=lane_url + entrypoint_query, + title=ancestor.display_name, + ) + ) + + # Append the breadcrumbs to the feed. + self._feed.breadcrumbs = breadcrumbs + + def show_current_entrypoint(self, entrypoint: Optional[Type[EntryPoint]]) -> None: + """Annotate this given feed with a simplified:entryPoint + attribute pointing to the current entrypoint's TYPE_URI. + + This gives clients an overall picture of the type of works in + the feed, and a way to distinguish between one EntryPoint + and another. + + :param entrypoint: An EntryPoint. + """ + if not entrypoint: + return None + + if not entrypoint.URI: + return None + self._feed.entrypoint = entrypoint.URI + + @classmethod + def error_message( + cls, identifier: Identifier, error_status: int, error_message: str + ) -> OPDSMessage: + """Turn an error result into an OPDSMessage suitable for + adding to a feed. + """ + return OPDSMessage(identifier.urn, error_status, error_message) + + # All feed generating classmethods below + # Each classmethod creates a different kind of feed + + @classmethod + def page( + cls, + _db: Session, + title: str, + url: str, + worklist: WorkList, + annotator: CirculationManagerAnnotator, + facets: Optional[FacetsWithEntryPoint], + pagination: Optional[Pagination], + search_engine: Optional[ExternalSearchIndex], + ) -> OPDSAcquisitionFeed: + works = worklist.works( + _db, facets=facets, pagination=pagination, search_engine=search_engine + ) + """A basic paged feed""" + # "works" MAY be a generator, we want a list + if not isinstance(works, list): + works = list(works) + + feed = OPDSAcquisitionFeed( + title, url, works, annotator, facets=facets, pagination=pagination + ) + + feed.generate_feed() + feed.add_pagination_links(works, worklist) + feed.add_facet_links(worklist) + + if isinstance(facets, FacetsWithEntryPoint): + feed.add_breadcrumb_links(worklist, facets.entrypoint) + + return feed + + @classmethod + def active_loans_for( + cls, + circulation: Optional[CirculationAPI], + patron: Patron, + annotator: Optional[LibraryAnnotator] = None, + **response_kwargs: Any, + ) -> OPDSAcquisitionFeed: + """A patron specific feed that only contains the loans and holds of a patron""" + db = Session.object_session(patron) + active_loans_by_work = {} + for loan in patron.loans: + work = loan.work + if work: + active_loans_by_work[work] = loan + + # There might be multiple holds for the same work so we gather all of them and choose the best one. + all_holds_by_work: Dict[Work, List[Hold]] = {} + for hold in patron.holds: + work = hold.work + if not work: + continue + + if work not in all_holds_by_work: + all_holds_by_work[work] = [] + + all_holds_by_work[work].append(hold) + + active_holds_by_work: Dict[Work, Hold] = {} + for work, list_of_holds in all_holds_by_work.items(): + active_holds_by_work[ + work + ] = LibraryLoanAndHoldAnnotator.choose_best_hold_for_work(list_of_holds) + + if not annotator: + annotator = LibraryLoanAndHoldAnnotator( + circulation, None, patron.library, patron + ) + + annotator.active_holds_by_work = active_holds_by_work + annotator.active_loans_by_work = active_loans_by_work + url = annotator.url_for( + "active_loans", library_short_name=patron.library.short_name, _external=True + ) + works = patron.works_on_loan_or_on_hold() + + feed = OPDSAcquisitionFeed("Active loans and holds", url, works, annotator) + feed.generate_feed() + return feed + + @classmethod + def single_entry_loans_feed( + cls, + circulation: Any, + item: LicensePool | Loan, + annotator: LibraryAnnotator | None = None, + fulfillment: FulfillmentInfo | None = None, + **response_kwargs: Any, + ) -> OPDSEntryResponse | ProblemDetail | None: + """A single entry as a standalone feed specific to a patron""" + if not item: + raise ValueError("Argument 'item' must be non-empty") + + if isinstance(item, LicensePool): + license_pool = item + library = circulation.library + elif isinstance(item, (Loan, Hold)): + license_pool = item.license_pool + library = item.library + else: + raise ValueError( + "Argument 'item' must be an instance of {}, {}, or {} classes".format( + Loan, Hold, LicensePool + ) + ) + + if not annotator: + annotator = LibraryLoanAndHoldAnnotator(circulation, None, library) + + log = logging.getLogger(cls.__name__) + + # Sometimes the pool or work may be None + # In those cases we have to protect against the exceptions + try: + work = license_pool.work or license_pool.presentation_edition.work + except AttributeError as ex: + log.error(f"Error retrieving a Work Object {ex}") + log.error( + f"Error Data: {license_pool} | {license_pool and license_pool.presentation_edition}" + ) + return NOT_FOUND_ON_REMOTE + + if not work: + return NOT_FOUND_ON_REMOTE + + _db = Session.object_session(item) + active_loans_by_work: Any = {} + active_holds_by_work: Any = {} + active_fulfillments_by_work = {} + item_dictionary = None + + if isinstance(item, Loan): + item_dictionary = active_loans_by_work + elif isinstance(item, Hold): + item_dictionary = active_holds_by_work + + if item_dictionary is not None: + item_dictionary[work] = item + + if fulfillment: + active_fulfillments_by_work[work] = fulfillment + + annotator.active_loans_by_work = active_loans_by_work + annotator.active_holds_by_work = active_holds_by_work + annotator.active_fulfillments_by_work = active_fulfillments_by_work + identifier = license_pool.identifier + + entry = cls.single_entry(work, annotator, even_if_no_license_pool=True) + + if isinstance(entry, WorkEntry) and entry.computed: + return cls.entry_as_response(entry, **response_kwargs) + elif isinstance(entry, OPDSMessage): + return cls.entry_as_response(entry, max_age=0) + + return None + + @classmethod + def single_entry( + cls, + work: Work | Edition | None, + annotator: Annotator, + even_if_no_license_pool: bool = False, + ) -> Optional[WorkEntry | OPDSMessage]: + """Turn a work into an annotated work entry for an acquisition feed.""" + identifier = None + _work: Work + if isinstance(work, Edition): + active_edition = work + identifier = active_edition.primary_identifier + active_license_pool = None + _work = active_edition.work # We always need a work for an entry + else: + if not work: + # We have a license pool but no work. Most likely we don't have + # metadata for this work yet. + return None + _work = work + active_license_pool = annotator.active_licensepool_for(work) + if active_license_pool: + identifier = active_license_pool.identifier + active_edition = active_license_pool.presentation_edition + elif work.presentation_edition: + active_edition = work.presentation_edition + identifier = active_edition.primary_identifier + + # There's no reason to present a book that has no active license pool. + if not identifier: + logging.warning("%r HAS NO IDENTIFIER", work) + return None + + if not active_license_pool and not even_if_no_license_pool: + logging.warning("NO ACTIVE LICENSE POOL FOR %r", work) + return cls.error_message( + identifier, + 403, + "I've heard about this work but have no active licenses for it.", + ) + + if not active_edition: + logging.warning("NO ACTIVE EDITION FOR %r", active_license_pool) + return cls.error_message( + identifier, + 403, + "I've heard about this work but have no metadata for it.", + ) + + try: + return cls._create_entry( + _work, active_license_pool, active_edition, identifier, annotator + ) + except UnfulfillableWork as e: + logging.info( + "Work %r is not fulfillable, refusing to create an .", + work, + ) + return cls.error_message( + identifier, + 403, + "I know about this work but can offer no way of fulfilling it.", + ) + except Exception as e: + logging.error("Exception generating OPDS entry for %r", work, exc_info=e) + return None + + @classmethod + def groups( + cls, + _db: Session, + title: str, + url: str, + worklist: WorkList, + annotator: LibraryAnnotator, + pagination: Optional[Pagination] = None, + facets: Optional[FacetsWithEntryPoint] = None, + search_engine: Optional[ExternalSearchIndex] = None, + search_debug: bool = False, + ) -> OPDSAcquisitionFeed: + """Internal method called by groups() when a grouped feed + must be regenerated. + """ + + # Try to get a set of (Work, WorkList) 2-tuples + # to make a normal grouped feed. + works_and_lanes = [ + x + for x in worklist.groups( + _db=_db, + pagination=pagination, + facets=facets, + search_engine=search_engine, + debug=search_debug, + ) + ] + # Make a typical grouped feed. + all_works = [] + for work, sublane in works_and_lanes: + if sublane == worklist: + # We are looking at the groups feed for (e.g.) + # "Science Fiction", and we're seeing a book + # that is featured within "Science Fiction" itself + # rather than one of the sublanes. + # + # We want to assign this work to a group called "All + # Science Fiction" and point its 'group URI' to + # the linear feed of the "Science Fiction" lane + # (as opposed to the groups feed, which is where we + # are now). + v = dict( + lane=worklist, + label=worklist.display_name_for_all, + link_to_list_feed=True, + ) + else: + # We are looking at the groups feed for (e.g.) + # "Science Fiction", and we're seeing a book + # that is featured within one of its sublanes, + # such as "Space Opera". + # + # We want to assign this work to a group derived + # from the sublane. + v = dict(lane=sublane) + + annotator.lanes_by_work[work].append(v) + all_works.append(work) + + feed = OPDSAcquisitionFeed( + title, url, all_works, annotator, facets=facets, pagination=pagination + ) + feed.generate_feed() + + # Regardless of whether or not the entries in feed can be + # grouped together, we want to apply certain feed-level + # annotations. + + # A grouped feed may link to alternate entry points into + # the data. + if facets: + entrypoints = facets.selectable_entrypoints(worklist) + if entrypoints: + + def make_link(ep: Type[EntryPoint]) -> str: + return annotator.groups_url( + worklist, facets=facets.navigate(entrypoint=ep) + ) + + cls.add_entrypoint_links( + feed._feed, make_link, entrypoints, facets.entrypoint + ) + + # A grouped feed may have breadcrumb links. + feed.add_breadcrumb_links(worklist, facets.entrypoint) + + return feed + + @classmethod + def search( + cls, + _db: Session, + title: str, + url: str, + lane: WorkList, + search_engine: ExternalSearchIndex, + query: str, + annotator: LibraryAnnotator, + pagination: Optional[Pagination] = None, + facets: Optional[FacetsWithEntryPoint] = None, + **response_kwargs: Any, + ) -> OPDSAcquisitionFeed | ProblemDetail: + """Run a search against the given search engine and return + the results as a Flask Response. + + :param _db: A database connection + :param title: The title of the resulting OPDS feed. + :param url: The URL from which the feed will be served. + :param search_engine: An ExternalSearchIndex. + :param query: The search query + :param pagination: A Pagination + :param facets: A Facets + :param annotator: An Annotator + :param response_kwargs: Keyword arguments to pass into the OPDSFeedResponse + constructor. + :return: An ODPSFeedResponse + """ + facets = facets or SearchFacets() + pagination = pagination or Pagination.default() + + try: + results = lane.search( + _db, query, search_engine, pagination=pagination, facets=facets + ) + except QueryParseException as e: + return INVALID_INPUT.detailed(e.detail) + + feed = OPDSAcquisitionFeed( + title, url, results, annotator, facets=facets, pagination=pagination + ) + feed.generate_feed() + feed.add_link( + annotator.default_lane_url(), rel="start", title=annotator.top_level_title() + ) + + # A feed of search results may link to alternate entry points + # into those results. + entrypoints = facets.selectable_entrypoints(lane) + if entrypoints: + + def make_link(ep: Type[EntryPoint]) -> str: + return annotator.search_url( + lane, query, pagination=None, facets=facets.navigate(entrypoint=ep) + ) + + cls.add_entrypoint_links( + feed._feed, + make_link, + entrypoints, + facets.entrypoint, + ) + + feed.add_pagination_links(results, lane) + + # Add "up" link. + feed.add_link( + annotator.lane_url(lane), + rel="up", + title=str(lane.display_name), + ) + + # We do not add breadcrumbs to this feed since you're not + # technically searching the this lane; you are searching the + # library's entire collection, using _some_ of the constraints + # imposed by this lane (notably language and audience). + return feed + + @classmethod + def from_query( + cls, + query: Query[Work], + _db: Session, + feed_name: str, + url: str, + pagination: Pagination, + url_fn: Callable[[int], str], + annotator: CirculationManagerAnnotator, + ) -> OPDSAcquisitionFeed: + """Build a feed representing one page of a given list. Currently used for + creating an OPDS feed for a custom list and not cached. + + TODO: This is used by the circulation manager admin interface. + Investigate changing the code that uses this to use the search + index -- this is inefficient and creates an alternate code path + that may harbor bugs. + + TODO: This cannot currently return OPDSFeedResponse because the + admin interface modifies the feed after it's generated. + + """ + page_of_works = pagination.modify_database_query(_db, query) + pagination.total_size = int(query.count()) + + feed = OPDSAcquisitionFeed( + feed_name, url, page_of_works, annotator, pagination=pagination + ) + feed.generate_feed(annotate=False) + + if pagination.total_size > 0 and pagination.has_next_page: + feed.add_link(url_fn(pagination.next_page.offset), rel="next") + if pagination.offset > 0: + feed.add_link(url_fn(pagination.first_page.offset), rel="first") + if pagination.previous_page: + feed.add_link( + url_fn(pagination.previous_page.offset), + rel="previous", + ) + + return feed + + +class LookupAcquisitionFeed(OPDSAcquisitionFeed): + """Used when the user has requested a lookup of a specific identifier, + which may be different from the identifier used by the Work's + default LicensePool. + """ + + @classmethod + def single_entry(cls, work: Tuple[Identifier, Work], annotator: Annotator) -> WorkEntry | OPDSMessage: # type: ignore[override] + # This comes in as a tuple, which deviates from the typical behaviour + identifier, _work = work + + active_licensepool: Optional[LicensePool] + if identifier.licensed_through: + active_licensepool = identifier.licensed_through[0] + else: + # Use the default active LicensePool for the Work. + active_licensepool = annotator.active_licensepool_for(_work) + + error_status = error_message = None + if not active_licensepool: + error_status = 404 + error_message = "Identifier not found in collection" + elif identifier.work != _work: + error_status = 500 + error_message = ( + 'I tried to generate an OPDS entry for the identifier "%s" using a Work not associated with that identifier.' + % identifier.urn + ) + + if error_status: + return cls.error_message(identifier, error_status, error_message or "") + + if active_licensepool: + edition = active_licensepool.presentation_edition + else: + edition = _work.presentation_edition + try: + return cls._create_entry( + _work, active_licensepool, edition, identifier, annotator + ) + except UnfulfillableWork as e: + logging.info( + "Work %r is not fulfillable, refusing to create an .", _work + ) + return cls.error_message( + identifier, + 403, + "I know about this work but can offer no way of fulfilling it.", + ) diff --git a/core/feed/admin.py b/core/feed/admin.py new file mode 100644 index 0000000000..a4536fa18e --- /dev/null +++ b/core/feed/admin.py @@ -0,0 +1,70 @@ +from typing import Optional + +from sqlalchemy import and_ +from sqlalchemy.orm import Session +from typing_extensions import Self + +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.admin import AdminAnnotator +from core.lane import Pagination +from core.model.licensing import LicensePool + + +class AdminFeed(OPDSAcquisitionFeed): + @classmethod + def suppressed( + cls, + _db: Session, + title: str, + url: str, + annotator: AdminAnnotator, + pagination: Optional[Pagination] = None, + ) -> Self: + _pagination = pagination or Pagination.default() + + q = ( + _db.query(LicensePool) + .filter( + and_( + LicensePool.suppressed == True, + LicensePool.superceded == False, + ) + ) + .order_by(LicensePool.id) + ) + pools = _pagination.modify_database_query(_db, q).all() + + works = [pool.work for pool in pools] + feed = cls(title, url, works, annotator, pagination=_pagination) + feed.generate_feed() + + # Render a 'start' link + top_level_title = annotator.top_level_title() + start_uri = annotator.groups_url(None) + + feed.add_link(start_uri, rel="start", title=top_level_title) + + # Render an 'up' link, same as the 'start' link to indicate top-level feed + feed.add_link(start_uri, rel="up", title=top_level_title) + + if len(works) > 0: + # There are works in this list. Add a 'next' link. + feed.add_link( + href=annotator.suppressed_url(_pagination.next_page), + rel="next", + ) + + if _pagination.offset > 0: + feed.add_link( + annotator.suppressed_url(_pagination.first_page), + rel="first", + ) + + previous_page = _pagination.previous_page + if previous_page: + feed.add_link( + annotator.suppressed_url(previous_page), + rel="previous", + ) + + return feed diff --git a/core/feed/annotator/admin.py b/core/feed/annotator/admin.py new file mode 100644 index 0000000000..193b8e70f2 --- /dev/null +++ b/core/feed/annotator/admin.py @@ -0,0 +1,105 @@ +from datetime import datetime +from typing import Optional + +from api.circulation import CirculationAPI +from core.feed.annotator.circulation import LibraryAnnotator +from core.feed.annotator.verbose import VerboseAnnotator +from core.feed.types import FeedData, Link, WorkEntry +from core.lane import Pagination +from core.mirror import MirrorUploader +from core.model import DataSource +from core.model.configuration import ExternalIntegrationLink +from core.model.library import Library + + +class AdminAnnotator(LibraryAnnotator): + def __init__(self, circulation: Optional[CirculationAPI], library: Library) -> None: + super().__init__(circulation, None, library) + + def annotate_work_entry( + self, entry: WorkEntry, updated: Optional[datetime] = None + ) -> None: + super().annotate_work_entry(entry) + if not entry.computed: + return + VerboseAnnotator.add_ratings(entry) + + identifier = entry.identifier + active_license_pool = entry.license_pool + + # Find staff rating and add a tag for it. + for measurement in identifier.measurements: + if ( + measurement.data_source.name == DataSource.LIBRARY_STAFF # type: ignore[attr-defined] + and measurement.is_most_recent + and measurement.value is not None + ): + entry.computed.ratings.append( + self.rating(measurement.quantity_measured, measurement.value) + ) + + if active_license_pool and active_license_pool.suppressed: + entry.computed.other_links.append( + Link( + href=self.url_for( + "unsuppress", + identifier_type=identifier.type, + identifier=identifier.identifier, + _external=True, + ), + rel="http://librarysimplified.org/terms/rel/restore", + ) + ) + else: + entry.computed.other_links.append( + Link( + href=self.url_for( + "suppress", + identifier_type=identifier.type, + identifier=identifier.identifier, + _external=True, + ), + rel="http://librarysimplified.org/terms/rel/hide", + ) + ) + + entry.computed.other_links.append( + Link( + href=self.url_for( + "edit", + identifier_type=identifier.type, + identifier=identifier.identifier, + _external=True, + ), + rel="edit", + ) + ) + + # If there is a storage integration for the collection, changing the cover is allowed. + if active_license_pool: + mirror = MirrorUploader.for_collection( + active_license_pool.collection, ExternalIntegrationLink.COVERS + ) + if mirror: + entry.computed.other_links.append( + Link( + href=self.url_for( + "work_change_book_cover", + identifier_type=identifier.type, + identifier=identifier.identifier, + _external=True, + ), + rel="http://librarysimplified.org/terms/rel/change_cover", + ) + ) + + def suppressed_url(self, pagination: Pagination) -> str: + kwargs = dict(list(pagination.items())) + return self.url_for("suppressed", _external=True, **kwargs) + + def annotate_feed(self, feed: FeedData) -> None: + # Add a 'search' link. + search_url = self.url_for("lane_search", languages=None, _external=True) + feed.add_link( + search_url, rel="search", type="application/opensearchdescription+xml" + ) diff --git a/core/feed/annotator/base.py b/core/feed/annotator/base.py new file mode 100644 index 0000000000..4ce50ea4fc --- /dev/null +++ b/core/feed/annotator/base.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import datetime +import logging +from collections import defaultdict +from decimal import Decimal +from typing import Any, Dict, List, Optional, Set, Tuple +from urllib.parse import quote + +from sqlalchemy.orm import Session, joinedload + +from core.classifier import Classifier +from core.feed.types import ( + Author, + FeedData, + FeedEntryType, + Link, + WorkEntry, + WorkEntryData, +) +from core.feed.util import strftime +from core.model.classification import Subject +from core.model.contributor import Contribution, Contributor +from core.model.datasource import DataSource +from core.model.edition import Edition +from core.model.library import Library +from core.model.licensing import LicensePool +from core.model.resource import Hyperlink +from core.model.work import Work +from core.util.opds_writer import AtomFeed, OPDSFeed + + +class ToFeedEntry: + @classmethod + def authors(cls, edition: Edition) -> Dict[str, List[Author]]: + """Create one or more author (and contributor) objects for the given + Work. + + :param edition: The Edition to use as a reference + for bibliographic information, including the list of + Contributions. + :return: A dict with "authors" and "contributors" as a list of Author objects + """ + authors: Dict[str, List[Author]] = {"authors": [], "contributors": []} + state: Dict[Optional[str], Set[str]] = defaultdict(set) + for contribution in edition.contributions: + info = cls.contributor(contribution, state) + if info is None: + # contributor_tag decided that this contribution doesn't + # need a tag. + continue + key, tag = info + authors[f"{key}s"].append(tag) + + if authors["authors"]: + return authors + + # We have no author information, so we add empty tag + # to avoid the implication (per RFC 4287 4.2.1) that this book + # was written by whoever wrote the OPDS feed. + authors["authors"].append(Author(name="")) + return authors + + @classmethod + def contributor( + cls, contribution: Contribution, state: Dict[Optional[str], Set[str]] + ) -> Optional[Tuple[str, Author]]: + """Build an author (or contributor) object for a Contribution. + + :param contribution: A Contribution. + :param state: A defaultdict of sets, which may be used to keep + track of what happened during previous calls to + contributor for a given Work. + :return: An Author object, or None if creating an Author for this Contribution + would be redundant or of low value. + + """ + contributor = contribution.contributor + role = contribution.role + current_role: str + + if role in Contributor.AUTHOR_ROLES: + current_role = "author" + marc_role = None + elif role is not None: + current_role = "contributor" + marc_role = Contributor.MARC_ROLE_CODES.get(role) + if not marc_role: + # This contribution is not one that we publish as + # a tag. Skip it. + return None + else: + return None + + name = contributor.display_name or contributor.sort_name + name_key = name.lower() + if name_key in state[marc_role]: + # We've already credited this person with this + # MARC role. Returning a tag would be redundant. + return None + + # Okay, we're creating a tag. + properties: Dict[str, Any] = dict() + if marc_role: + properties["role"] = marc_role + entry = Author(name=name, **properties) + + # Record the fact that we credited this person with this role, + # so that we don't do it again on a subsequent call. + state[marc_role].add(name_key) + + return current_role, entry + + @classmethod + def series( + cls, series_name: Optional[str], series_position: Optional[int] | Optional[str] + ) -> Optional[FeedEntryType]: + """Generate a FeedEntryType object for the given name and position.""" + if not series_name: + return None + series_details = dict() + series_details["name"] = series_name + if series_position != None: + series_details["position"] = str(series_position) + series = FeedEntryType.create(**series_details) + return series + + @classmethod + def rating(cls, type_uri: Optional[str], value: float | Decimal) -> FeedEntryType: + """Generate a FeedEntryType object for the given type and value.""" + entry = FeedEntryType.create( + **dict(ratingValue="%.4f" % value, additionalType=type_uri) + ) + return entry + + @classmethod + def samples(cls, edition: Optional[Edition]) -> list[Hyperlink]: + if not edition: + return [] + _db = Session.object_session(edition) + links = ( + _db.query(Hyperlink) + .filter( + Hyperlink.rel == Hyperlink.SAMPLE, + Hyperlink.identifier_id == edition.primary_identifier_id, + ) + .options(joinedload(Hyperlink.resource)) + .all() + ) + return links + + @classmethod + def categories(cls, work: Work) -> Dict[str, List[Dict[str, str]]]: + """Return all relevant classifications of this work. + + :return: A dictionary mapping 'scheme' URLs to dictionaries of + attribute-value pairs. + + Notable attributes: 'term', 'label', 'ratingValue' + """ + if not work: + return {} + + categories = {} + + fiction_term = None + if work.fiction == True: + fiction_term = "Fiction" + elif work.fiction == False: + fiction_term = "Nonfiction" + if fiction_term: + fiction_scheme = Subject.SIMPLIFIED_FICTION_STATUS + categories[fiction_scheme] = [ + dict(term=fiction_scheme + fiction_term, label=fiction_term) + ] + + simplified_genres = [] + for wg in work.work_genres: + simplified_genres.append(wg.genre.name) # type: ignore[attr-defined] + + if simplified_genres: + categories[Subject.SIMPLIFIED_GENRE] = [ + dict(term=Subject.SIMPLIFIED_GENRE + quote(x), label=x) + for x in simplified_genres + ] + + # Add the appeals as a category of schema + # http://librarysimplified.org/terms/appeal + schema_url = AtomFeed.SIMPLIFIED_NS + "appeals/" + appeals: List[Dict[str, Any]] = [] + categories[schema_url] = appeals + for name, value in ( + (Work.CHARACTER_APPEAL, work.appeal_character), + (Work.LANGUAGE_APPEAL, work.appeal_language), + (Work.SETTING_APPEAL, work.appeal_setting), + (Work.STORY_APPEAL, work.appeal_story), + ): + if value: + appeal: Dict[str, Any] = dict(term=schema_url + name, label=name) + weight_field = "ratingValue" + appeal[weight_field] = value + appeals.append(appeal) + + # Add the audience as a category of schema + # http://schema.org/audience + if work.audience: + audience_uri = "http://schema.org/audience" + categories[audience_uri] = [dict(term=work.audience, label=work.audience)] + + # Any book can have a target age, but the target age + # is only relevant for childrens' and YA books. + audiences_with_target_age = ( + Classifier.AUDIENCE_CHILDREN, + Classifier.AUDIENCE_YOUNG_ADULT, + ) + if work.target_age and work.audience in audiences_with_target_age: + uri = Subject.uri_lookup[Subject.AGE_RANGE] + target_age = work.target_age_string + if target_age: + categories[uri] = [dict(term=target_age, label=target_age)] + + return categories + + @classmethod + def content(cls, work: Optional[Work]) -> str: + """Return an HTML summary of this work.""" + summary = "" + if work: + if work.summary_text is not None: + summary = work.summary_text + elif ( + work.summary + and work.summary.representation + and work.summary.representation.content + ): + content = work.summary.representation.content + if isinstance(content, bytes): + content = content.decode("utf-8") + work.summary_text = content + summary = work.summary_text + return summary + + +class Annotator(ToFeedEntry): + def annotate_work_entry( + self, entry: WorkEntry, updated: Optional[datetime.datetime] = None + ) -> None: + """ + Any data that the serializer must consider while generating an "entry" + must be populated in this method. + The serializer may not use all the data populated based on the protocol it is bound to. + """ + if entry.computed: + return + + work = entry.work + edition = entry.edition + identifier = entry.identifier + pool = entry.license_pool + computed = WorkEntryData() + + image_links = [] + other_links = [] + for rel, url in [ + (Hyperlink.IMAGE, work.cover_full_url), + (Hyperlink.THUMBNAIL_IMAGE, work.cover_thumbnail_url), + ]: + if not url: + continue + image_type = "image/png" + if url.endswith(".jpeg") or url.endswith(".jpg"): + image_type = "image/jpeg" + elif url.endswith(".gif"): + image_type = "image/gif" + image_links.append(Link(rel=rel, href=url, type=image_type)) + + samples = self.samples(edition) + for sample in samples: + other_links.append( + Link( + rel=Hyperlink.CLIENT_SAMPLE, + href=sample.resource.url, + type=sample.resource.representation.media_type, + ) + ) + + if edition.medium: + additional_type = Edition.medium_to_additional_type.get(str(edition.medium)) + if not additional_type: + logging.warning("No additionalType for medium %s", edition.medium) + computed.additionalType = additional_type + + computed.title = FeedEntryType(text=(edition.title or OPDSFeed.NO_TITLE)) + + if edition.subtitle: + computed.subtitle = FeedEntryType(text=edition.subtitle) + if edition.sort_title: + computed.sort_title = FeedEntryType(text=edition.sort_title) + + author_entries = self.authors(edition) + computed.contributors = author_entries.get("contributors", []) + computed.authors = author_entries.get("authors", []) + + if edition.series: + computed.series = self.series(edition.series, edition.series_position) + + content = self.content(work) + if content: + computed.summary = FeedEntryType(text=content) + computed.summary.add_attributes(dict(type="html")) + + computed.pwid = edition.permanent_work_id + + categories_by_scheme = self.categories(work) + category_tags = [] + for scheme, categories in list(categories_by_scheme.items()): + for category in categories: + category = dict( + list(map(str, (k, v))) for k, v in list(category.items()) + ) + category_tag = FeedEntryType.create(scheme=scheme, **category) + category_tags.append(category_tag) + computed.categories = category_tags + + if edition.language_code: + computed.language = FeedEntryType(text=edition.language_code) + + if edition.publisher: + computed.publisher = FeedEntryType(text=edition.publisher) + + if edition.imprint: + computed.imprint = FeedEntryType(text=edition.imprint) + + if edition.issued or edition.published: + computed.issued = edition.issued or edition.published + + if identifier: + computed.identifier = identifier.urn + + if pool: + data_source = pool.data_source.name + if data_source != DataSource.INTERNAL_PROCESSING: + # INTERNAL_PROCESSING indicates a dummy LicensePool + # created as a stand-in, e.g. by the metadata wrangler. + # This component is not actually distributing the book, + # so it should not have a bibframe:distribution tag. + computed.distribution = FeedEntryType() + computed.distribution.add_attributes(dict(provider_name=data_source)) + + # We use Atom 'published' for the date the book first became + # available to people using this application. + avail = pool.availability_time + if avail: + today = datetime.date.today() + if isinstance(avail, datetime.datetime): + avail_date = avail.date() + else: + avail_date = avail # type: ignore[unreachable] + if avail_date <= today: # Avoid obviously wrong values. + computed.published = FeedEntryType(text=strftime(avail_date)) + + if not updated and entry.work.last_update_time: + # NOTE: This is a default that works in most cases. When + # ordering Opensearch results by last update time, + # `work` is a WorkSearchResult object containing a more + # reliable value that you can use if you want. + updated = entry.work.last_update_time + if updated: + computed.updated = FeedEntryType(text=strftime(updated)) + + computed.image_links = image_links + computed.other_links = other_links + entry.computed = computed + + def annotate_feed(self, feed: FeedData) -> None: + """Any additional metadata or links that should be added to the feed (not each entry) + should be added to the FeedData object in this method. + """ + + def active_licensepool_for( + self, work: Work, library: Library | None = None + ) -> LicensePool | None: + """Which license pool would be/has been used to issue a license for + this work? + """ + if not work: + return None + + return work.active_license_pool(library=library) diff --git a/core/feed/annotator/circulation.py b/core/feed/annotator/circulation.py new file mode 100644 index 0000000000..1538f253b4 --- /dev/null +++ b/core/feed/annotator/circulation.py @@ -0,0 +1,1564 @@ +from __future__ import annotations + +import copy +import datetime +import logging +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple + +from flask import url_for +from sqlalchemy.orm import Session + +from api.adobe_vendor_id import AuthdataUtility +from api.annotations import AnnotationWriter +from api.circulation import BaseCirculationAPI, CirculationAPI +from api.config import Configuration +from api.lanes import DynamicLane +from api.novelist import NoveListAPI +from core.analytics import Analytics +from core.classifier import Classifier +from core.config import CannotLoadConfiguration +from core.entrypoint import EverythingEntryPoint +from core.external_search import WorkSearchResult +from core.feed.annotator.base import Annotator +from core.feed.types import ( + Acquisition, + FeedData, + FeedEntryType, + IndirectAcquisition, + Link, + WorkEntry, +) +from core.feed.util import strftime +from core.lane import Facets, FacetsWithEntryPoint, Lane, Pagination, WorkList +from core.lcp.credential import LCPCredentialFactory, LCPHashedPassphrase +from core.lcp.exceptions import LCPError +from core.model.circulationevent import CirculationEvent +from core.model.collection import Collection +from core.model.edition import Edition +from core.model.formats import FormatPriorities +from core.model.identifier import Identifier +from core.model.integration import IntegrationConfiguration +from core.model.library import Library +from core.model.licensing import ( + DeliveryMechanism, + LicensePool, + LicensePoolDeliveryMechanism, +) +from core.model.patron import Hold, Loan, Patron +from core.model.work import Work +from core.opds import UnfulfillableWork +from core.util.datetime_helpers import from_timestamp +from core.util.opds_writer import OPDSFeed + + +class AcquisitionHelper: + @classmethod + def license_tags( + cls, + license_pool: Optional[LicensePool], + loan: Optional[Loan], + hold: Optional[Hold], + ) -> Optional[Dict[str, Any]]: + acquisition = {} + # Generate a list of licensing tags. These should be inserted + # into a tag. + status = None + since = None + until = None + + if not license_pool: + return None + default_loan_period = default_reservation_period = None + collection = license_pool.collection + obj: Loan | Hold + if (loan or hold) and not license_pool.open_access: + if loan: + obj = loan + elif hold: + obj = hold + default_loan_period = datetime.timedelta( + collection.default_loan_period(obj.library) + ) + if loan: + status = "available" + since = loan.start + if not loan.license_pool.unlimited_access: + until = loan.until(default_loan_period) + elif hold: + if not license_pool.open_access: + default_reservation_period = datetime.timedelta( + collection.default_reservation_period + ) + until = hold.until(default_loan_period, default_reservation_period) + if hold.position == 0: + status = "ready" + since = None + else: + status = "reserved" + since = hold.start + elif ( + license_pool.open_access + or license_pool.unlimited_access + or (license_pool.licenses_available > 0 and license_pool.licenses_owned > 0) + ): + status = "available" + else: + status = "unavailable" + + acquisition["availability_status"] = status + if since: + acquisition["availability_since"] = strftime(since) + if until: + acquisition["availability_until"] = strftime(until) + + # Open-access pools do not need to display or . + if license_pool.open_access or license_pool.unlimited_access: + return acquisition + + total = license_pool.patrons_in_hold_queue or 0 + + if hold: + if hold.position is None: + # This shouldn't happen, but if it does, assume we're last + # in the list. + position = total + else: + position = hold.position + + if position > 0: + acquisition["holds_position"] = str(position) + if position > total: + # The patron's hold position appears larger than the total + # number of holds. This happens frequently because the + # number of holds and a given patron's hold position are + # updated by different processes. Don't propagate this + # appearance to the client. + total = position + elif position == 0 and total == 0: + # The book is reserved for this patron but they're not + # counted as having it on hold. This is the only case + # where we know that the total number of holds is + # *greater* than the hold position. + total = 1 + acquisition["holds_total"] = str(total) + + acquisition["copies_total"] = str(license_pool.licenses_owned or 0) + acquisition["copies_available"] = str(license_pool.licenses_available or 0) + + return acquisition + + @classmethod + def format_types(cls, delivery_mechanism: DeliveryMechanism) -> List[str]: + """Generate a set of types suitable for passing into + acquisition_link(). + """ + types = [] + # If this is a streaming book, you have to get an OPDS entry, then + # get a direct link to the streaming reader from that. + if delivery_mechanism.is_streaming: + types.append(OPDSFeed.ENTRY_TYPE) + + # If this is a DRM-encrypted book, you have to get through the DRM + # to get the goodies inside. + drm = delivery_mechanism.drm_scheme_media_type + if drm: + types.append(drm) + + # Finally, you get the goodies. + media = delivery_mechanism.content_type_media_type + if media: + types.append(media) + + return types + + +class CirculationManagerAnnotator(Annotator): + hidden_content_types: list[str] + + def __init__( + self, + lane: Optional[WorkList], + active_loans_by_work: Optional[Dict[Work, Loan]] = None, + active_holds_by_work: Optional[Dict[Work, Hold]] = None, + active_fulfillments_by_work: Optional[Dict[Work, Any]] = None, + hidden_content_types: Optional[List[str]] = None, + ) -> None: + if lane: + logger_name = "Circulation Manager Annotator for %s" % lane.display_name + else: + logger_name = "Circulation Manager Annotator" + self.log = logging.getLogger(logger_name) + self.lane = lane + self.active_loans_by_work = active_loans_by_work or {} + self.active_holds_by_work = active_holds_by_work or {} + self.active_fulfillments_by_work = active_fulfillments_by_work or {} + self.hidden_content_types = hidden_content_types or [] + self.facet_view = "feed" + + def is_work_entry_solo(self, work: Work) -> bool: + """Return a boolean value indicating whether the work's OPDS catalog entry is served by itself, + rather than as a part of the feed. + + :param work: Work object + :type work: core.model.work.Work + + :return: Boolean value indicating whether the work's OPDS catalog entry is served by itself, + rather than as a part of the feed + :rtype: bool + """ + return any( + work in x # type: ignore[operator] # Mypy gets confused with complex "in" statements + for x in ( + self.active_loans_by_work, + self.active_holds_by_work, + self.active_fulfillments_by_work, + ) + ) + + def _lane_identifier(self, lane: Optional[WorkList]) -> Optional[int]: + if isinstance(lane, Lane): + return lane.id + return None + + def top_level_title(self) -> str: + return "" + + def default_lane_url(self) -> str: + return self.feed_url(None) + + def lane_url(self, lane: WorkList) -> str: + return self.feed_url(lane) + + def url_for(self, *args: Any, **kwargs: Any) -> str: + return url_for(*args, **kwargs) + + def facet_url(self, facets: Facets) -> str: + return self.feed_url(self.lane, facets=facets, default_route=self.facet_view) + + def feed_url( + self, + lane: Optional[WorkList], + facets: Optional[FacetsWithEntryPoint] = None, + pagination: Optional[Pagination] = None, + default_route: str = "feed", + extra_kwargs: Optional[Dict[str, Any]] = None, + ) -> str: + if isinstance(lane, WorkList) and hasattr(lane, "url_arguments"): + route, kwargs = lane.url_arguments + else: + route = default_route + lane_identifier = self._lane_identifier(lane) + kwargs = dict(lane_identifier=lane_identifier) + if facets is not None: + kwargs.update(dict(list(facets.items()))) + if pagination is not None: + kwargs.update(dict(list(pagination.items()))) + if extra_kwargs: + kwargs.update(extra_kwargs) + return self.url_for(route, _external=True, **kwargs) + + def navigation_url(self, lane: Lane) -> str: + return self.url_for( + "navigation_feed", + lane_identifier=self._lane_identifier(lane), + library_short_name=lane.library.short_name, + _external=True, + ) + + def active_licensepool_for( + self, work: Work, library: Optional[Library] = None + ) -> Optional[LicensePool]: + loan = self.active_loans_by_work.get(work) or self.active_holds_by_work.get( + work + ) + if loan: + # The active license pool is the one associated with + # the loan/hold. + return loan.license_pool + else: + # There is no active loan. Use the default logic for + # determining the active license pool. + return super().active_licensepool_for(work, library=library) + + @staticmethod + def _prioritized_formats_for_pool( + licensepool: LicensePool, + ) -> tuple[list[str], list[str]]: + collection: Collection = licensepool.collection + config: IntegrationConfiguration = collection.integration_configuration + + # Consult the configuration information for the integration configuration + # that underlies the license pool's collection. The configuration + # information _might_ contain a set of prioritized DRM schemes and + # content types. + prioritized_drm_schemes: list[str] = ( + config.settings_dict.get(FormatPriorities.PRIORITIZED_DRM_SCHEMES_KEY) or [] + ) + + content_setting: List[str] = ( + config.settings_dict.get(FormatPriorities.PRIORITIZED_CONTENT_TYPES_KEY) + or [] + ) + return prioritized_drm_schemes, content_setting + + @staticmethod + def _deprioritized_lcp_content( + licensepool: LicensePool, + ) -> bool: + collection: Collection = licensepool.collection + config: IntegrationConfiguration = collection.integration_configuration + + # Consult the configuration information for the integration configuration + # that underlies the license pool's collection. The configuration + # information _might_ contain a flag that indicates whether to deprioritize + # LCP content. By default, if no configuration value is specified, then + # the priority of LCP content will be left completely unchanged. + + _prioritize: bool = config.settings_dict.get( + FormatPriorities.DEPRIORITIZE_LCP_NON_EPUBS_KEY, False + ) + return _prioritize + + def visible_delivery_mechanisms( + self, licensepool: LicensePool | None + ) -> list[LicensePoolDeliveryMechanism]: + if not licensepool: + return [] + + ( + prioritized_drm_schemes, + prioritized_content_types, + ) = CirculationManagerAnnotator._prioritized_formats_for_pool(licensepool) + + return FormatPriorities( + prioritized_drm_schemes=prioritized_drm_schemes, + prioritized_content_types=prioritized_content_types, + hidden_content_types=self.hidden_content_types, + deprioritize_lcp_non_epubs=CirculationManagerAnnotator._deprioritized_lcp_content( + licensepool + ), + ).prioritize_for_pool(licensepool) + + def annotate_work_entry( + self, + entry: WorkEntry, + updated: Optional[datetime.datetime] = None, + ) -> None: + work = entry.work + identifier = entry.identifier or work.presentation_edition.primary_identifier + active_license_pool = entry.license_pool or self.active_licensepool_for(work) + # If OpenSearch included a more accurate last_update_time, + # use it instead of Work.last_update_time + updated = entry.work.last_update_time + if isinstance(work, WorkSearchResult): + # Opensearch puts this field in a list, but we've set it up + # so there will be at most one value. + last_updates = getattr(work._hit, "last_update", []) + if last_updates: + # last_update is seconds-since epoch; convert to UTC datetime. + updated = from_timestamp(last_updates[0]) + + # There's a chance that work.last_updated has been + # modified but the change hasn't made it to the search + # engine yet. Even then, we stick with the search + # engine value, because a sorted list is more + # important to the import process than an up-to-date + # 'last update' value. + + super().annotate_work_entry(entry, updated=updated) + active_loan = self.active_loans_by_work.get(work) + active_hold = self.active_holds_by_work.get(work) + active_fulfillment = self.active_fulfillments_by_work.get(work) + + # Now we need to generate a tag for every delivery mechanism + # that has well-defined media types. + link_tags = self.acquisition_links( + active_license_pool, + active_loan, + active_hold, + active_fulfillment, + identifier, + ) + if entry.computed: + for tag in link_tags: + entry.computed.acquisition_links.append(tag) + + def acquisition_links( + self, + active_license_pool: Optional[LicensePool], + active_loan: Optional[Loan], + active_hold: Optional[Hold], + active_fulfillment: Optional[Any], + identifier: Identifier, + can_hold: bool = True, + can_revoke_hold: bool = True, + set_mechanism_at_borrow: bool = False, + direct_fulfillment_delivery_mechanisms: Optional[ + List[LicensePoolDeliveryMechanism] + ] = None, + add_open_access_links: bool = True, + ) -> List[Acquisition]: + """Generate a number of tags that enumerate all acquisition + methods. + + :param direct_fulfillment_delivery_mechanisms: A way to + fulfill each LicensePoolDeliveryMechanism in this list will be + presented as a link with + rel="http://opds-spec.org/acquisition/open-access", indicating + that it can be downloaded with no intermediate steps such as + authentication. + """ + can_borrow = False + can_fulfill = False + can_revoke = False + + if active_loan: + can_fulfill = True + can_revoke = True + elif active_hold: + # We display the borrow link even if the patron can't + # borrow the book right this minute. + can_borrow = True + + can_revoke = can_revoke_hold + elif active_fulfillment: + can_fulfill = True + can_revoke = True + else: + # The patron has no existing relationship with this + # work. Give them the opportunity to check out the work + # or put it on hold. + can_borrow = True + + # If there is something to be revoked for this book, + # add a link to revoke it. + revoke_links = [] + if active_license_pool and can_revoke: + revoke_links.append( + self.revoke_link(active_license_pool, active_loan, active_hold) + ) + + # Add next-step information for every useful delivery + # mechanism. + borrow_links = [] + if can_borrow: + # Borrowing a book gives you an OPDS entry that gives you + # fulfillment links for every visible delivery mechanism. + visible_mechanisms = self.visible_delivery_mechanisms(active_license_pool) + if set_mechanism_at_borrow and active_license_pool: + # The ebook distributor requires that the delivery + # mechanism be set at the point of checkout. This means + # a separate borrow link for each mechanism. + for mechanism in visible_mechanisms: + borrow_links.append( + self.borrow_link( + active_license_pool, mechanism, [mechanism], active_hold + ) + ) + elif active_license_pool: + # The ebook distributor does not require that the + # delivery mechanism be set at the point of + # checkout. This means a single borrow link with + # indirectAcquisition tags for every visible delivery + # mechanism. If a delivery mechanism must be set, it + # will be set at the point of fulfillment. + borrow_links.append( + self.borrow_link( + active_license_pool, None, visible_mechanisms, active_hold + ) + ) + + # Generate the licensing tags that tell you whether the book + # is available. + for link in borrow_links: + if link is not None: + license_tags = AcquisitionHelper.license_tags( + active_license_pool, active_loan, active_hold + ) + if license_tags is not None: + link.add_attributes(license_tags) + + # Add links for fulfilling an active loan. + fulfill_links: List[Optional[Acquisition]] = [] + if can_fulfill: + if active_fulfillment: + # We're making an entry for a specific fulfill link. + type = active_fulfillment.content_type + url = active_fulfillment.content_link + rel = OPDSFeed.ACQUISITION_REL + link_tag = self.acquisition_link( + rel=rel, href=url, types=[type], active_loan=active_loan + ) + fulfill_links.append(link_tag) + + elif active_loan and active_loan.fulfillment and active_license_pool: + # The delivery mechanism for this loan has been + # set. There is one link for the delivery mechanism + # that was locked in, and links for any streaming + # delivery mechanisms. + # + # Since the delivery mechanism has already been locked in, + # we choose not to use visible_delivery_mechanisms -- + # they already chose it and they're stuck with it. + for lpdm in active_license_pool.delivery_mechanisms: + if ( + lpdm is active_loan.fulfillment + or lpdm.delivery_mechanism.is_streaming + ): + fulfill_links.append( + self.fulfill_link( + active_license_pool, + active_loan, + lpdm.delivery_mechanism, + ) + ) + elif active_license_pool is not None: + # The delivery mechanism for this loan has not been + # set. There is one fulfill link for every visible + # delivery mechanism. + for lpdm in self.visible_delivery_mechanisms(active_license_pool): + fulfill_links.append( + self.fulfill_link( + active_license_pool, active_loan, lpdm.delivery_mechanism + ) + ) + + open_access_links: List[Optional[Acquisition]] = [] + if ( + active_license_pool is not None + and direct_fulfillment_delivery_mechanisms is not None + ): + for lpdm in direct_fulfillment_delivery_mechanisms: + # These links use the OPDS 'open-access' link relation not + # because they are open access in the licensing sense, but + # because they are ways to download the book "without any + # requirement, which includes payment and registration." + # + # To avoid confusion, we explicitly add a dc:rights + # statement to each link explaining what the rights are to + # this title. + direct_fulfill = self.fulfill_link( + active_license_pool, + active_loan, + lpdm.delivery_mechanism, + rel=OPDSFeed.OPEN_ACCESS_REL, + ) + if direct_fulfill: + direct_fulfill.add_attributes(self.rights_attributes(lpdm)) + open_access_links.append(direct_fulfill) + + # If this is an open-access book, add an open-access link for + # every delivery mechanism with an associated resource. + # But only if this library allows it, generally this is if + # a library has no patron authentication attached to it + if ( + add_open_access_links + and active_license_pool + and active_license_pool.open_access + ): + for lpdm in active_license_pool.delivery_mechanisms: + if lpdm.resource: + open_access_links.append( + self.open_access_link(active_license_pool, lpdm) + ) + + return [ + x + for x in borrow_links + fulfill_links + open_access_links + revoke_links + if x is not None + ] + + def revoke_link( + self, + active_license_pool: LicensePool, + active_loan: Optional[Loan], + active_hold: Optional[Hold], + ) -> Optional[Acquisition]: + return None + + def borrow_link( + self, + active_license_pool: LicensePool, + borrow_mechanism: Optional[LicensePoolDeliveryMechanism], + fulfillment_mechanisms: List[LicensePoolDeliveryMechanism], + active_hold: Optional[Hold] = None, + ) -> Optional[Acquisition]: + return None + + def fulfill_link( + self, + license_pool: LicensePool, + active_loan: Optional[Loan], + delivery_mechanism: DeliveryMechanism, + rel: str = OPDSFeed.ACQUISITION_REL, + ) -> Optional[Acquisition]: + return None + + def open_access_link( + self, pool: LicensePool, lpdm: LicensePoolDeliveryMechanism + ) -> Acquisition: + kw: Dict[str, Any] = dict(rel=OPDSFeed.OPEN_ACCESS_REL, type="") + + # Start off assuming that the URL associated with the + # LicensePoolDeliveryMechanism's Resource is the URL we should + # send for download purposes. This will be the case unless we + # previously mirrored that URL somewhere else. + href = lpdm.resource.url + + rep = lpdm.resource.representation + if rep: + if rep.media_type: + kw["type"] = rep.media_type + href = rep.public_url + kw["href"] = href + kw.update(self.rights_attributes(lpdm)) + link = Acquisition(**kw) + link.availability_status = "available" + return link + + def rights_attributes( + self, lpdm: Optional[LicensePoolDeliveryMechanism] + ) -> Dict[str, str]: + """Create a dictionary of tag attributes that explain the + rights status of a LicensePoolDeliveryMechanism. + + If nothing is known, the dictionary will be empty. + """ + if not lpdm or not lpdm.rights_status or not lpdm.rights_status.uri: + return {} + rights_attr = "rights" + return {rights_attr: lpdm.rights_status.uri} + + @classmethod + def acquisition_link( + cls, + rel: str, + href: str, + types: Optional[List[str]], + active_loan: Optional[Loan] = None, + ) -> Acquisition: + if types: + initial_type = types[0] + indirect_types = types[1:] + else: + initial_type = None + indirect_types = [] + link = Acquisition( + href=href, + rel=rel, + type=initial_type, + is_loan=True if active_loan else False, + ) + indirect = cls.indirect_acquisition(indirect_types) + + if indirect is not None: + link.indirect_acquisitions = [indirect] + return link + + @classmethod + def indirect_acquisition( + cls, indirect_types: List[str] + ) -> Optional[IndirectAcquisition]: + top_level_parent: Optional[IndirectAcquisition] = None + parent: Optional[IndirectAcquisition] = None + for t in indirect_types: + indirect_link = IndirectAcquisition(type=t) + if parent is not None: + parent.children = [indirect_link] + parent = indirect_link + if top_level_parent is None: + top_level_parent = indirect_link + return top_level_parent + + +class LibraryAnnotator(CirculationManagerAnnotator): + def __init__( + self, + circulation: Optional[CirculationAPI], + lane: Optional[WorkList], + library: Library, + patron: Optional[Patron] = None, + active_loans_by_work: Optional[Dict[Work, Loan]] = None, + active_holds_by_work: Optional[Dict[Work, Hold]] = None, + active_fulfillments_by_work: Optional[Dict[Work, Any]] = None, + facet_view: str = "feed", + top_level_title: str = "All Books", + library_identifies_patrons: bool = True, + facets: Optional[FacetsWithEntryPoint] = None, + ) -> None: + """Constructor. + + :param library_identifies_patrons: A boolean indicating + whether or not this library can distinguish between its + patrons. A library might not authenticate patrons at + all, or it might distinguish patrons from non-patrons in a + way that does not allow it to keep track of individuals. + + If this is false, links that imply the library can + distinguish between patrons will not be included. Depending + on the configured collections, some extra links may be + added, for direct acquisition of titles that would normally + require a loan. + """ + super().__init__( + lane, + active_loans_by_work=active_loans_by_work, + active_holds_by_work=active_holds_by_work, + active_fulfillments_by_work=active_fulfillments_by_work, + hidden_content_types=library.settings.hidden_content_types, + ) + self.circulation = circulation + self.library: Library = library + self.patron = patron + self.lanes_by_work: Dict[Work, List[Any]] = defaultdict(list) + self.facet_view = facet_view + self._adobe_id_cache: Dict[str, Any] = {} + self._top_level_title = top_level_title + self.identifies_patrons = library_identifies_patrons + self.facets = facets or None + + def top_level_title(self) -> str: + return self._top_level_title + + def permalink_for(self, identifier: Identifier) -> Tuple[str, str]: + # TODO: Do not force OPDS types + url = self.url_for( + "permalink", + identifier_type=identifier.type, + identifier=identifier.identifier, + library_short_name=self.library.short_name, + _external=True, + ) + return url, OPDSFeed.ENTRY_TYPE + + def groups_url( + self, lane: Optional[WorkList], facets: Optional[FacetsWithEntryPoint] = None + ) -> str: + lane_identifier = self._lane_identifier(lane) + if facets: + kwargs = dict(list(facets.items())) + else: + kwargs = {} + + return self.url_for( + "acquisition_groups", + lane_identifier=lane_identifier, + library_short_name=self.library.short_name, + _external=True, + **kwargs, + ) + + def default_lane_url(self, facets: Optional[FacetsWithEntryPoint] = None) -> str: + return self.groups_url(None, facets=facets) + + def feed_url( # type: ignore [override] + self, + lane: Optional[WorkList], + facets: Optional[FacetsWithEntryPoint] = None, + pagination: Optional[Pagination] = None, + default_route: str = "feed", + ) -> str: + extra_kwargs = dict() + if self.library: + extra_kwargs["library_short_name"] = self.library.short_name + return super().feed_url(lane, facets, pagination, default_route, extra_kwargs) + + def search_url( + self, + lane: Optional[WorkList], + query: str, + pagination: Optional[Pagination], + facets: Optional[FacetsWithEntryPoint] = None, + ) -> str: + lane_identifier = self._lane_identifier(lane) + kwargs = dict(q=query) + if facets: + kwargs.update(dict(list(facets.items()))) + if pagination: + kwargs.update(dict(list(pagination.items()))) + return self.url_for( + "lane_search", + lane_identifier=lane_identifier, + library_short_name=self.library.short_name, + _external=True, + **kwargs, + ) + + def group_uri( + self, work: Work, license_pool: Optional[LicensePool], identifier: Identifier + ) -> Tuple[Optional[str], str]: + if not work in self.lanes_by_work: + return None, "" + + lanes = self.lanes_by_work[work] + if not lanes: + # I don't think this should ever happen? + lane_name = None + url = self.url_for( + "acquisition_groups", + lane_identifier=None, + library_short_name=self.library.short_name, + _external=True, + ) + title = "All Books" + return url, title + + lane = lanes[0] + self.lanes_by_work[work] = lanes[1:] + lane_name = "" + show_feed = False + + if isinstance(lane, dict): + show_feed = lane.get("link_to_list_feed", show_feed) + title = lane.get("label", lane_name) + lane = lane["lane"] + + if isinstance(lane, str): + return lane, lane_name + + if hasattr(lane, "display_name") and not title: + title = lane.display_name + + if show_feed: + return self.feed_url(lane, self.facets), title + + return self.lane_url(lane, self.facets), title + + def lane_url( + self, lane: Optional[WorkList], facets: Optional[FacetsWithEntryPoint] = None + ) -> str: + # If the lane has sublanes, the URL identifying the group will + # take the user to another set of groups for the + # sublanes. Otherwise it will take the user to a list of the + # books in the lane by author. + + if lane and isinstance(lane, Lane) and lane.sublanes: + url = self.groups_url(lane, facets=facets) + elif lane and (isinstance(lane, Lane) or isinstance(lane, DynamicLane)): + url = self.feed_url(lane, facets) + else: + # This lane isn't part of our lane hierarchy. It's probably + # a WorkList created to represent the top-level. Use the top-level + # url for it. + url = self.default_lane_url(facets=facets) + return url + + def annotate_work_entry( + self, entry: WorkEntry, updated: Optional[datetime.datetime] = None + ) -> None: + super().annotate_work_entry(entry, updated=updated) + + if not entry.computed: + return + + work = entry.work + identifier = entry.identifier or work.presentation_edition.primary_identifier + + permalink_uri, permalink_type = self.permalink_for(identifier) + # TODO: Do not force OPDS types + if permalink_uri: + entry.computed.other_links.append( + Link(href=permalink_uri, rel="alternate", type=permalink_type) + ) + if self.is_work_entry_solo(work): + entry.computed.other_links.append( + Link(rel="self", href=permalink_uri, type=permalink_type) + ) + + # Add a link to each author tag. + self.add_author_links(entry) + + # And a series, if there is one. + if work.series: + self.add_series_link(entry) + + if NoveListAPI.is_configured(self.library): + # If NoveList Select is configured, there might be + # recommendations, too. + entry.computed.other_links.append( + Link( + rel="recommendations", + type=OPDSFeed.ACQUISITION_FEED_TYPE, + title="Recommended Works", + href=self.url_for( + "recommendations", + identifier_type=identifier.type, + identifier=identifier.identifier, + library_short_name=self.library.short_name, + _external=True, + ), + ) + ) + + # Add a link for related books if available. + if self.related_books_available(work, self.library): + entry.computed.other_links.append( + Link( + rel="related", + type=OPDSFeed.ACQUISITION_FEED_TYPE, + title="Recommended Works", + href=self.url_for( + "related_books", + identifier_type=identifier.type, + identifier=identifier.identifier, + library_short_name=self.library.short_name, + _external=True, + ), + ) + ) + + # Add a link to get a patron's annotations for this book. + if self.identifies_patrons: + entry.computed.other_links.append( + Link( + rel="http://www.w3.org/ns/oa#annotationService", + type=AnnotationWriter.CONTENT_TYPE, + href=self.url_for( + "annotations_for_work", + identifier_type=identifier.type, + identifier=identifier.identifier, + library_short_name=self.library.short_name, + _external=True, + ), + ) + ) + + if Analytics.is_configured(self.library): + entry.computed.other_links.append( + Link( + rel="http://librarysimplified.org/terms/rel/analytics/open-book", + href=self.url_for( + "track_analytics_event", + identifier_type=identifier.type, + identifier=identifier.identifier, + event_type=CirculationEvent.OPEN_BOOK, + library_short_name=self.library.short_name, + _external=True, + ), + ) + ) + + # Groups is only from the library annotator + group_uri, group_title = self.group_uri( + entry.work, entry.license_pool, entry.identifier + ) + if group_uri: + entry.computed.other_links.append( + Link(href=group_uri, rel=OPDSFeed.GROUP_REL, title=str(group_title)) + ) + + @classmethod + def related_books_available(cls, work: Work, library: Library) -> bool: + """:return: bool asserting whether related books might exist for a particular Work""" + contributions = work.sort_author and work.sort_author != Edition.UNKNOWN_AUTHOR + + return bool(contributions or work.series or NoveListAPI.is_configured(library)) + + def language_and_audience_key_from_work( + self, work: Work + ) -> Tuple[Optional[str], Optional[str]]: + language_key = work.language + + audiences = None + if work.audience == Classifier.AUDIENCE_CHILDREN: + audiences = [Classifier.AUDIENCE_CHILDREN] + elif work.audience == Classifier.AUDIENCE_YOUNG_ADULT: + audiences = Classifier.AUDIENCES_JUVENILE + elif work.audience == Classifier.AUDIENCE_ALL_AGES: + audiences = [Classifier.AUDIENCE_CHILDREN, Classifier.AUDIENCE_ALL_AGES] + elif work.audience in Classifier.AUDIENCES_ADULT: + audiences = list(Classifier.AUDIENCES_NO_RESEARCH) + elif work.audience == Classifier.AUDIENCE_RESEARCH: + audiences = list(Classifier.AUDIENCES) + else: + audiences = [] + + audience_key = None + if audiences: + audience_strings = [urllib.parse.quote_plus(a) for a in sorted(audiences)] + audience_key = ",".join(audience_strings) + + return language_key, audience_key + + def add_author_links(self, entry: WorkEntry) -> None: + """Add a link to all authors""" + if not entry.computed: + return None + + languages, audiences = self.language_and_audience_key_from_work(entry.work) + for author_entry in entry.computed.authors: + if not (name := getattr(author_entry, "name", None)): + continue + + author_entry.add_attributes( + { + "link": Link( + rel="contributor", + type=OPDSFeed.ACQUISITION_FEED_TYPE, + title=name, + href=self.url_for( + "contributor", + contributor_name=name, + languages=languages, + audiences=audiences, + library_short_name=self.library.short_name, + _external=True, + ), + ), + } + ) + + def add_series_link(self, entry: WorkEntry) -> None: + if not entry.computed: + return None + + series_entry = entry.computed.series + work = entry.work + + if series_entry is None: + # There is no series, and thus nothing to annotate. + # This probably indicates an out-of-date OPDS entry. + work_id = work.id + work_title = work.title + self.log.error( + 'add_series_link() called on work %s ("%s"), which has no Series data in its OPDS WorkEntry.', + work_id, + work_title, + ) + return + + series_name = work.series + languages, audiences = self.language_and_audience_key_from_work(work) + href = self.url_for( + "series", + series_name=series_name, + languages=languages, + audiences=audiences, + library_short_name=self.library.short_name, + _external=True, + ) + series_entry.add_attributes( + { + "link": Link( + rel="series", + type=OPDSFeed.ACQUISITION_FEED_TYPE, + title=series_name, + href=href, + ), + } + ) + + def annotate_feed(self, feed: FeedData) -> None: + if self.patron: + # A patron is authenticated. + self.add_patron(feed) + else: + # No patron is authenticated. Show them how to + # authenticate (or that authentication is not supported). + self.add_authentication_document_link(feed) + + # Add a 'search' link if the lane is searchable. + if self.lane and self.lane.search_target: + search_facet_kwargs = {} + if self.facets is not None: + if self.facets.entrypoint_is_default: + # The currently selected entry point is a default. + # Rather than using it, we want the 'default' behavior + # for search, which is to search everything. + search_facets = self.facets.navigate( + entrypoint=EverythingEntryPoint + ) + else: + search_facets = self.facets + search_facet_kwargs.update(dict(list(search_facets.items()))) + + lane_identifier = self._lane_identifier(self.lane) + search_url = self.url_for( + "lane_search", + lane_identifier=lane_identifier, + library_short_name=self.library.short_name, + _external=True, + **search_facet_kwargs, + ) + search_link = dict( + rel="search", + type="application/opensearchdescription+xml", + href=search_url, + ) + feed.add_link(**search_link) + + if self.identifies_patrons: + # Since this library authenticates patrons it can offer + # a bookshelf and an annotation service. + shelf_link = dict( + rel="http://opds-spec.org/shelf", + type=OPDSFeed.ACQUISITION_FEED_TYPE, + href=self.url_for( + "active_loans", + library_short_name=self.library.short_name, + _external=True, + ), + ) + feed.add_link(**shelf_link) + + annotations_link = dict( + rel="http://www.w3.org/ns/oa#annotationService", + type=AnnotationWriter.CONTENT_TYPE, + href=self.url_for( + "annotations", + library_short_name=self.library.short_name, + _external=True, + ), + ) + feed.add_link(**annotations_link) + + if self.lane and self.lane.uses_customlists: + name = None + if hasattr(self.lane, "customlists") and len(self.lane.customlists) == 1: + name = self.lane.customlists[0].name + else: + _db = Session.object_session(self.library) + customlist = self.lane.get_customlists(_db) + if customlist: + name = customlist[0].name + + if name: + crawlable_url = self.url_for( + "crawlable_list_feed", + list_name=name, + library_short_name=self.library.short_name, + _external=True, + ) + crawlable_link = dict( + rel="http://opds-spec.org/crawlable", + type=OPDSFeed.ACQUISITION_FEED_TYPE, + href=crawlable_url, + ) + feed.add_link(**crawlable_link) + + self.add_configuration_links(feed) + + def add_configuration_links(self, feed: FeedData) -> None: + _db = Session.object_session(self.library) + + def _add_link(l: Dict[str, Any]) -> None: + feed.add_link(**l) + + library = self.library + if library.settings.terms_of_service: + _add_link( + dict( + rel="terms-of-service", + href=library.settings.terms_of_service, + type="text/html", + ) + ) + + if library.settings.privacy_policy: + _add_link( + dict( + rel="privacy-policy", + href=library.settings.privacy_policy, + type="text/html", + ) + ) + + if library.settings.copyright: + _add_link( + dict( + rel="copyright", + href=library.settings.copyright, + type="text/html", + ) + ) + + if library.settings.about: + _add_link( + dict( + rel="about", + href=library.settings.about, + type="text/html", + ) + ) + + if library.settings.license: + _add_link( + dict( + rel="license", + href=library.settings.license, + type="text/html", + ) + ) + + navigation_urls = self.library.settings.web_header_links + navigation_labels = self.library.settings.web_header_labels + for url, label in zip(navigation_urls, navigation_labels): + d = dict( + href=url, + title=label, + type="text/html", + rel="related", + role="navigation", + ) + _add_link(d) + + for type, value in Configuration.help_uris(self.library): + d = dict(href=value, rel="help") + if type: + d["type"] = type + _add_link(d) + + def acquisition_links( # type: ignore [override] + self, + active_license_pool: Optional[LicensePool], + active_loan: Optional[Loan], + active_hold: Optional[Hold], + active_fulfillment: Optional[Any], + identifier: Identifier, + direct_fulfillment_delivery_mechanisms: Optional[ + List[LicensePoolDeliveryMechanism] + ] = None, + mock_api: Optional[Any] = None, + ) -> List[Acquisition]: + """Generate one or more tags that can be used to borrow, + reserve, or fulfill a book, depending on the state of the book + and the current patron. + + :param active_license_pool: The LicensePool for which we're trying to + generate tags. + :param active_loan: A Loan object representing the current patron's + existing loan for this title, if any. + :param active_hold: A Hold object representing the current patron's + existing hold on this title, if any. + :param active_fulfillment: A LicensePoolDeliveryMechanism object + representing the mechanism, if any, which the patron has chosen + to fulfill this work. + :param feed: The OPDSFeed that will eventually contain these + tags. + :param identifier: The Identifier of the title for which we're + trying to generate tags. + :param direct_fulfillment_delivery_mechanisms: A list of + LicensePoolDeliveryMechanisms for the given LicensePool + that should have fulfillment-type tags generated for + them, even if this method wouldn't normally think that + makes sense. + :param mock_api: A mock object to stand in for the API to the + vendor who provided this LicensePool. If this is not provided, a + live API for that vendor will be used. + """ + direct_fulfillment_delivery_mechanisms = ( + direct_fulfillment_delivery_mechanisms or [] + ) + api = mock_api + if not api and self.circulation and active_license_pool: + api = self.circulation.api_for_license_pool(active_license_pool) + if api: + set_mechanism_at_borrow = ( + api.SET_DELIVERY_MECHANISM_AT == BaseCirculationAPI.BORROW_STEP + ) + if active_license_pool and not self.identifies_patrons and not active_loan: + for lpdm in active_license_pool.delivery_mechanisms: + if api.can_fulfill_without_loan(None, active_license_pool, lpdm): + # This title can be fulfilled without an + # active loan, so we're going to add an acquisition + # link that goes directly to the fulfillment step + # without the 'borrow' step. + direct_fulfillment_delivery_mechanisms.append(lpdm) + else: + # This is most likely an open-access book. Just put one + # borrow link and figure out the rest later. + set_mechanism_at_borrow = False + + return super().acquisition_links( + active_license_pool, + active_loan, + active_hold, + active_fulfillment, + identifier, + can_hold=self.library.settings.allow_holds, + can_revoke_hold=bool( + active_hold + and ( + not self.circulation + or ( + active_license_pool + and self.circulation.can_revoke_hold( + active_license_pool, active_hold + ) + ) + ) + ), + set_mechanism_at_borrow=set_mechanism_at_borrow, + direct_fulfillment_delivery_mechanisms=direct_fulfillment_delivery_mechanisms, + add_open_access_links=(not self.identifies_patrons), + ) + + def revoke_link( + self, + active_license_pool: LicensePool, + active_loan: Optional[Loan], + active_hold: Optional[Hold], + ) -> Optional[Acquisition]: + if not self.identifies_patrons: + return None + url = self.url_for( + "revoke_loan_or_hold", + license_pool_id=active_license_pool.id, + library_short_name=self.library.short_name, + _external=True, + ) + kw: Dict[str, Any] = dict(href=url, rel=OPDSFeed.REVOKE_LOAN_REL) + revoke_link_tag = Acquisition(**kw) + return revoke_link_tag + + def borrow_link( + self, + active_license_pool: LicensePool, + borrow_mechanism: Optional[LicensePoolDeliveryMechanism], + fulfillment_mechanisms: List[LicensePoolDeliveryMechanism], + active_hold: Optional[Hold] = None, + ) -> Optional[Acquisition]: + if not self.identifies_patrons: + return None + identifier = active_license_pool.identifier + if borrow_mechanism: + # Following this link will both borrow the book and set + # its delivery mechanism. + mechanism_id = borrow_mechanism.delivery_mechanism.id + else: + # Following this link will borrow the book but not set + # its delivery mechanism. + mechanism_id = None + borrow_url = self.url_for( + "borrow", + identifier_type=identifier.type, + identifier=identifier.identifier, + mechanism_id=mechanism_id, + library_short_name=self.library.short_name, + _external=True, + ) + rel = OPDSFeed.BORROW_REL + borrow_link = Acquisition( + rel=rel, + href=borrow_url, + type=OPDSFeed.ENTRY_TYPE, + is_hold=True if active_hold else False, + ) + + indirect_acquisitions: List[IndirectAcquisition] = [] + for lpdm in fulfillment_mechanisms: + # We have information about one or more delivery + # mechanisms that will be available at the point of + # fulfillment. To the extent possible, put information + # about these mechanisms into the tag as + # tags. + + # These are the formats mentioned in the indirect + # acquisition. + format_types = AcquisitionHelper.format_types(lpdm.delivery_mechanism) + + # If we can borrow this book, add this delivery mechanism + # to the borrow link as an . + if format_types: + indirect_acquisition = self.indirect_acquisition(format_types) + if indirect_acquisition: + indirect_acquisitions.append(indirect_acquisition) + + if not indirect_acquisitions: + # If there's no way to actually get the book, cancel the creation + # of an OPDS entry altogether. + raise UnfulfillableWork() + + borrow_link.indirect_acquisitions = indirect_acquisitions + return borrow_link + + def fulfill_link( + self, + license_pool: LicensePool, + active_loan: Optional[Loan], + delivery_mechanism: DeliveryMechanism, + rel: str = OPDSFeed.ACQUISITION_REL, + ) -> Optional[Acquisition]: + """Create a new fulfillment link. + + This link may include tags from the OPDS Extensions for DRM. + """ + if not self.identifies_patrons and rel != OPDSFeed.OPEN_ACCESS_REL: + return None + if isinstance(delivery_mechanism, LicensePoolDeliveryMechanism): + logging.warning( + "LicensePoolDeliveryMechanism passed into fulfill_link instead of DeliveryMechanism!" + ) + delivery_mechanism = delivery_mechanism.delivery_mechanism + format_types = AcquisitionHelper.format_types(delivery_mechanism) + if not format_types: + return None + + fulfill_url = self.url_for( + "fulfill", + license_pool_id=license_pool.id, + mechanism_id=delivery_mechanism.id, + library_short_name=self.library.short_name, + _external=True, + ) + + link_tag = self.acquisition_link( + rel=rel, href=fulfill_url, types=format_types, active_loan=active_loan + ) + + children = AcquisitionHelper.license_tags(license_pool, active_loan, None) + if children: + link_tag.add_attributes(children) + + drm_tags = self.drm_extension_tags( + license_pool, active_loan, delivery_mechanism + ) + link_tag.add_attributes(drm_tags) + return link_tag + + def open_access_link( + self, pool: LicensePool, lpdm: LicensePoolDeliveryMechanism + ) -> Acquisition: + link_tag = super().open_access_link(pool, lpdm) + fulfill_url = self.url_for( + "fulfill", + license_pool_id=pool.id, + mechanism_id=lpdm.delivery_mechanism.id, + library_short_name=self.library.short_name, + _external=True, + ) + link_tag.href = fulfill_url + return link_tag + + def drm_extension_tags( + self, + license_pool: LicensePool, + active_loan: Optional[Loan], + delivery_mechanism: Optional[DeliveryMechanism], + ) -> Dict[str, Any]: + """Construct OPDS Extensions for DRM tags that explain how to + register a device with the DRM server that manages this loan. + :param delivery_mechanism: A DeliveryMechanism + """ + if not active_loan or not delivery_mechanism or not self.identifies_patrons: + return {} + + if delivery_mechanism.drm_scheme == DeliveryMechanism.ADOBE_DRM: + # Get an identifier for the patron that will be registered + # with the DRM server. + patron = active_loan.patron + + # Generate a tag that can feed into the + # Vendor ID service. + return self.adobe_id_tags(patron) + + if delivery_mechanism.drm_scheme == DeliveryMechanism.LCP_DRM: + # Generate a tag that can be used for the loan + # in the mobile apps. + + return self.lcp_key_retrieval_tags(active_loan) + + return {} + + def adobe_id_tags( + self, patron_identifier: str | Patron + ) -> Dict[str, FeedEntryType]: + """Construct tags using the DRM Extensions for OPDS standard that + explain how to get an Adobe ID for this patron, and how to + manage their list of device IDs. + :param delivery_mechanism: A DeliveryMechanism + :return: If Adobe Vendor ID delegation is configured, a list + containing a tag. If not, an empty list. + """ + # CirculationManagerAnnotators are created per request. + # Within the context of a single request, we can cache the + # tags that explain how the patron can get an Adobe ID, and + # reuse them across tags. This saves a little time, + # makes tests more reliable, and stops us from providing a + # different Short Client Token for every tag. + if isinstance(patron_identifier, Patron): + cache_key = str(patron_identifier.id) + else: + cache_key = patron_identifier + cached = self._adobe_id_cache.get(cache_key) + if cached is None: + cached = {} + authdata = None + try: + authdata = AuthdataUtility.from_config(self.library) + except CannotLoadConfiguration as e: + logging.error( + "Cannot load Short Client Token configuration; outgoing OPDS entries will not have DRM autodiscovery support", + exc_info=e, + ) + return {} + if authdata: + vendor_id, token = authdata.short_client_token_for_patron( + patron_identifier + ) + drm_licensor = FeedEntryType.create( + vendor=vendor_id, clientToken=FeedEntryType(text=token) + ) + cached = {"licensor": drm_licensor} + + self._adobe_id_cache[cache_key] = cached + else: + cached = copy.deepcopy(cached) + return cached + + def lcp_key_retrieval_tags(self, active_loan: Loan) -> Dict[str, FeedEntryType]: + # In the case of LCP we have to include a patron's hashed passphrase + # inside the acquisition link so client applications can use it to open the LCP license + # without having to ask the user to enter their password + # https://readium.org/lcp-specs/notes/lcp-key-retrieval.html#including-a-hashed-passphrase-in-an-opds-1-catalog + + db = Session.object_session(active_loan) + lcp_credential_factory = LCPCredentialFactory() + + response = {} + + try: + hashed_passphrase: LCPHashedPassphrase = ( + lcp_credential_factory.get_hashed_passphrase(db, active_loan.patron) + ) + response["hashed_passphrase"] = FeedEntryType(text=hashed_passphrase.hashed) + except LCPError: + # The patron's passphrase wasn't generated yet and not present in the database. + pass + + return response + + def add_patron(self, feed: FeedData) -> None: + if not self.patron or not self.identifies_patrons: + return None + patron_details = {} + if self.patron.username: + patron_details["username"] = self.patron.username + if self.patron.authorization_identifier: + patron_details[ + "authorizationIdentifier" + ] = self.patron.authorization_identifier + + patron_tag = FeedEntryType.create(**patron_details) + feed.add_metadata("patron", patron_tag) + + def add_authentication_document_link(self, feed_obj: FeedData) -> None: + """Create a tag that points to the circulation + manager's Authentication for OPDS document + for the current library. + """ + # Even if self.identifies_patrons is false, we include this link, + # because this document is the one that explains there is no + # patron authentication at this library. + feed_obj.add_link( + rel="http://opds-spec.org/auth/document", + href=self.url_for( + "authentication_document", + library_short_name=self.library.short_name, + _external=True, + ), + ) diff --git a/core/feed/annotator/loan_and_hold.py b/core/feed/annotator/loan_and_hold.py new file mode 100644 index 0000000000..efdf42977a --- /dev/null +++ b/core/feed/annotator/loan_and_hold.py @@ -0,0 +1,125 @@ +import copy +from datetime import datetime +from typing import Any, Dict, List, Optional + +from core.feed.types import FeedData, Link, WorkEntry +from core.model.configuration import ExternalIntegration +from core.model.constants import EditionConstants, LinkRelations +from core.model.patron import Hold, Patron + +from .circulation import LibraryAnnotator + + +class LibraryLoanAndHoldAnnotator(LibraryAnnotator): + @staticmethod + def choose_best_hold_for_work(list_of_holds: List[Hold]) -> Hold: + # We don't want holds that are connected to license pools without any licenses owned. Also, we want hold that + # would result in the least wait time for the patron. + + best = list_of_holds[0] + + for hold in list_of_holds: + # We don't want holds with LPs with 0 licenses owned. + if hold.license_pool.licenses_owned == 0: + continue + + # Our current hold's LP owns some licenses but maybe the best one wasn't changed yet. + if best.license_pool.licenses_owned == 0: + best = hold + continue + + # Since these numbers are updated by different processes there might be situation where we don't have + # all data filled out. + hold_position = ( + hold.position or hold.license_pool.patrons_in_hold_queue or 0 + ) + best_position = ( + best.position or best.license_pool.patrons_in_hold_queue or 0 + ) + + # Both the best hold and current hold own some licenses, try to figure out which one is better. + if ( + hold_position / hold.license_pool.licenses_owned + < best_position / best.license_pool.licenses_owned + ): + best = hold + + return best + + def drm_device_registration_feed_tags(self, patron: Patron) -> Dict[str, Any]: + """Return tags that provide information on DRM device deregistration + independent of any particular loan. These tags will go under + the tag. + + This allows us to deregister an Adobe ID, in preparation for + logout, even if there is no active loan that requires one. + """ + tags = copy.deepcopy(self.adobe_id_tags(patron)) + attr = "scheme" + for tag, value in tags.items(): + value.add_attributes( + {attr: "http://librarysimplified.org/terms/drm/scheme/ACS"} + ) + return tags + + @property + def user_profile_management_protocol_link(self) -> Link: + """Create a tag that points to the circulation + manager's User Profile Management Protocol endpoint + for the current patron. + """ + return Link( + rel="http://librarysimplified.org/terms/rel/user-profile", + href=self.url_for( + "patron_profile", + library_short_name=self.library.short_name, + _external=True, + ), + ) + + def annotate_feed(self, feed: FeedData) -> None: + """Annotate the feed with top-level DRM device registration tags + and a link to the User Profile Management Protocol endpoint. + """ + super().annotate_feed(feed) + if self.patron: + tags = self.drm_device_registration_feed_tags(self.patron) + link = self.user_profile_management_protocol_link + if link.href is not None: + feed.add_link(link.href, rel=link.rel) + for name, value in tags.items(): + feed.add_metadata(name, feed_entry=value) + + def annotate_work_entry( + self, entry: WorkEntry, updated: Optional[datetime] = None + ) -> None: + super().annotate_work_entry(entry, updated=updated) + if not entry.computed: + return + active_license_pool = entry.license_pool + work = entry.work + edition = work.presentation_edition + identifier = edition.primary_identifier + # Only OPDS for Distributors should get the time tracking link + # And only if there is an active loan for the work + if ( + edition.medium == EditionConstants.AUDIO_MEDIUM + and active_license_pool + and active_license_pool.collection.protocol + == ExternalIntegration.OPDS_FOR_DISTRIBUTORS + and work in self.active_loans_by_work + ): + entry.computed.other_links.append( + Link( + rel=LinkRelations.TIME_TRACKING, + href=self.url_for( + "track_playtime_events", + identifier_type=identifier.type, + identifier=identifier.identifier, + library_short_name=self.library.short_name, + collection_id=active_license_pool.collection.id, + _external=True, + ), + type="application/json", + ) + ) diff --git a/core/feed/annotator/verbose.py b/core/feed/annotator/verbose.py new file mode 100644 index 0000000000..965f36aa01 --- /dev/null +++ b/core/feed/annotator/verbose.py @@ -0,0 +1,110 @@ +from collections import defaultdict +from datetime import datetime +from typing import Dict, List, Optional + +from sqlalchemy.orm import Session + +from core.feed.annotator.base import Annotator +from core.feed.types import Author, WorkEntry +from core.model import PresentationCalculationPolicy +from core.model.classification import Subject +from core.model.contributor import Contributor +from core.model.edition import Edition +from core.model.identifier import Identifier +from core.model.measurement import Measurement +from core.model.work import Work + + +class VerboseAnnotator(Annotator): + """The default Annotator for machine-to-machine integration. + + This Annotator describes all categories and authors for the book + in great detail. + """ + + opds_cache_field = Work.verbose_opds_entry.name + + def annotate_work_entry( + self, entry: WorkEntry, updated: Optional[datetime] = None + ) -> None: + super().annotate_work_entry(entry, updated=updated) + self.add_ratings(entry) + + @classmethod + def add_ratings(cls, entry: WorkEntry) -> None: + """Add a quality rating to the work.""" + work = entry.work + for type_uri, value in [ + (Measurement.QUALITY, work.quality), + (None, work.rating), + (Measurement.POPULARITY, work.popularity), + ]: + if value and entry.computed: + entry.computed.ratings.append(cls.rating(type_uri, value)) + + @classmethod + def categories( + cls, work: Work, policy: Optional[PresentationCalculationPolicy] = None + ) -> Dict[str, List[Dict[str, str]]]: + """Send out _all_ categories for the work. + + (So long as the category type has a URI associated with it in + Subject.uri_lookup.) + + :param policy: A PresentationCalculationPolicy to + use when deciding how deep to go when finding equivalent + identifiers for the work. + """ + policy = policy or PresentationCalculationPolicy( + equivalent_identifier_cutoff=100 + ) + _db = Session.object_session(work) + by_scheme_and_term = dict() + identifier_ids = work.all_identifier_ids(policy=policy) + classifications = Identifier.classifications_for_identifier_ids( + _db, identifier_ids + ) + for c in classifications: + subject = c.subject + if subject.type in Subject.uri_lookup: + scheme = Subject.uri_lookup[subject.type] + term = subject.identifier + weight_field = "ratingValue" + key = (scheme, term) + if not key in by_scheme_and_term: + value = dict(term=subject.identifier) + if subject.name: + value["label"] = subject.name + value[weight_field] = 0 + by_scheme_and_term[key] = value + by_scheme_and_term[key][weight_field] += c.weight + + # Collapse by_scheme_and_term to by_scheme + by_scheme = defaultdict(list) + for (scheme, term), value in list(by_scheme_and_term.items()): + by_scheme[scheme].append(value) + by_scheme.update(super().categories(work)) + return by_scheme + + @classmethod + def authors(cls, edition: Edition) -> Dict[str, List[Author]]: + """Create a detailed tag for each author.""" + return { + "authors": [ + cls.detailed_author(author) for author in edition.author_contributors + ], + "contributors": [], + } + + @classmethod + def detailed_author(cls, contributor: Contributor) -> Author: + """Turn a Contributor into a detailed tag.""" + author = Author() + author.name = contributor.display_name + author.sort_name = contributor.sort_name + author.family_name = contributor.family_name + author.wikipedia_name = contributor.wikipedia_name + author.viaf = f"http://viaf.org/viaf/{contributor.viaf}" + author.lc = f"http://id.loc.gov/authorities/names/{contributor.lc}" + + return author diff --git a/core/feed/base.py b/core/feed/base.py new file mode 100644 index 0000000000..d731e49244 --- /dev/null +++ b/core/feed/base.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod + +from flask import Response + + +class FeedInterface(ABC): + @abstractmethod + def generate_feed(self) -> None: + ... + + @abstractmethod + def as_response(self) -> Response: + ... diff --git a/core/feed/navigation.py b/core/feed/navigation.py new file mode 100644 index 0000000000..8c4031cc01 --- /dev/null +++ b/core/feed/navigation.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from typing import Any, Optional + +from sqlalchemy.orm import Session +from typing_extensions import Self +from werkzeug.datastructures import MIMEAccept + +from core.feed.annotator.circulation import CirculationManagerAnnotator +from core.feed.opds import BaseOPDSFeed +from core.feed.types import DataEntry, DataEntryTypes, Link +from core.feed.util import strftime +from core.lane import Facets, Pagination, WorkList +from core.opds import NavigationFacets +from core.util.datetime_helpers import utc_now +from core.util.flask_util import OPDSFeedResponse +from core.util.opds_writer import OPDSFeed + + +class NavigationFeed(BaseOPDSFeed): + def __init__( + self, + title: str, + url: str, + lane: WorkList, + annotator: CirculationManagerAnnotator, + facets: Optional[Facets] = None, + pagination: Optional[Pagination] = None, + ) -> None: + self.lane = lane + self.annotator = annotator + self._facets = facets + self._pagination = pagination + super().__init__(title, url) + + @classmethod + def navigation( + cls, + _db: Session, + title: str, + url: str, + worklist: WorkList, + annotator: CirculationManagerAnnotator, + facets: Optional[Facets] = None, + ) -> Self: + """The navigation feed with links to a given lane's sublanes.""" + + facets = facets or NavigationFacets.default(worklist) + feed = cls(title, url, worklist, annotator, facets=facets) + feed.generate_feed() + return feed + + def generate_feed(self) -> None: + self._feed.add_metadata("title", text=self.title) + self._feed.add_metadata("id", text=self.url) + self._feed.add_metadata("updated", text=strftime(utc_now())) + self._feed.add_link(href=self.url, rel="self") + if not self.lane.children: + # We can't generate links to children, since this Worklist + # has no children, so we'll generate a link to the + # Worklist's page-type feed instead. + title = "All " + self.lane.display_name + page_url = self.annotator.feed_url(self.lane) + self.add_entry(page_url, title, OPDSFeed.ACQUISITION_FEED_TYPE) + + for child in self.lane.visible_children: + title = child.display_name + if child.children: + child_url = self.annotator.navigation_url(child) + self.add_entry(child_url, title, OPDSFeed.NAVIGATION_FEED_TYPE) + else: + child_url = self.annotator.feed_url(child) + self.add_entry(child_url, title, OPDSFeed.ACQUISITION_FEED_TYPE) + + self.annotator.annotate_feed(self._feed) + + def add_entry( + self, url: str, title: str, type: str = OPDSFeed.NAVIGATION_FEED_TYPE + ) -> None: + """Create an OPDS navigation entry for a URL.""" + entry = DataEntry(type=DataEntryTypes.NAVIGATION, title=title, id=url) + entry.links.append(Link(rel="subsection", href=url, type=type)) + self._feed.data_entries.append(entry) + + def as_response( + self, + mime_types: Optional[MIMEAccept] = None, + **kwargs: Any, + ) -> OPDSFeedResponse: + response = super().as_response(mime_types=mime_types, **kwargs) + response.content_type = OPDSFeed.NAVIGATION_FEED_TYPE + return response diff --git a/core/feed/opds.py b/core/feed/opds.py new file mode 100644 index 0000000000..84fe5246dc --- /dev/null +++ b/core/feed/opds.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, Type + +from werkzeug.datastructures import MIMEAccept + +from core.feed.base import FeedInterface +from core.feed.serializer.base import SerializerInterface +from core.feed.serializer.opds import OPDS1Serializer +from core.feed.serializer.opds2 import OPDS2Serializer +from core.feed.types import FeedData, WorkEntry +from core.util.flask_util import OPDSEntryResponse, OPDSFeedResponse +from core.util.opds_writer import OPDSMessage + + +def get_serializer( + mime_types: Optional[MIMEAccept], +) -> SerializerInterface[Any]: + # Ordering matters for poor matches (eg. */*), so we will keep OPDS1 first + serializers: Dict[str, Type[SerializerInterface[Any]]] = { + "application/atom+xml": OPDS1Serializer, + "application/opds+json": OPDS2Serializer, + } + if mime_types: + match = mime_types.best_match( + serializers.keys(), default="application/atom+xml" + ) + return serializers[match]() + # Default + return OPDS1Serializer() + + +class BaseOPDSFeed(FeedInterface): + def __init__( + self, + title: str, + url: str, + precomposed_entries: Optional[List[OPDSMessage]] = None, + ) -> None: + self.url = url + self.title = title + self._precomposed_entries = precomposed_entries or [] + self._feed = FeedData() + self.log = logging.getLogger(self.__class__.__name__) + + def serialize(self, mime_types: Optional[MIMEAccept] = None) -> bytes: + serializer = get_serializer(mime_types) + return serializer.serialize_feed(self._feed) + + def add_link(self, href: str, rel: Optional[str] = None, **kwargs: Any) -> None: + self._feed.add_link(href, rel=rel, **kwargs) + + def as_response( + self, + mime_types: Optional[MIMEAccept] = None, + **kwargs: Any, + ) -> OPDSFeedResponse: + """Serialize the feed using the serializer protocol""" + serializer = get_serializer(mime_types) + return OPDSFeedResponse( + serializer.serialize_feed( + self._feed, precomposed_entries=self._precomposed_entries + ), + content_type=serializer.content_type(), + **kwargs, + ) + + @classmethod + def entry_as_response( + cls, + entry: WorkEntry | OPDSMessage, + mime_types: Optional[MIMEAccept] = None, + **response_kwargs: Any, + ) -> OPDSEntryResponse: + serializer = get_serializer(mime_types) + if isinstance(entry, OPDSMessage): + return OPDSEntryResponse( + response=serializer.to_string(serializer.serialize_opds_message(entry)), + status=entry.status_code, + content_type=serializer.content_type(), + **response_kwargs, + ) + + # A WorkEntry + if not entry.computed: + logging.getLogger().error(f"Entry data has not been generated for {entry}") + raise ValueError(f"Entry data has not been generated") + response = OPDSEntryResponse( + response=serializer.serialize_work_entry(entry.computed), + **response_kwargs, + ) + if isinstance(serializer, OPDS2Serializer): + # Only OPDS2 has the same content type for feed and entry + response.content_type = serializer.content_type() + return response diff --git a/core/feed/serializer/base.py b/core/feed/serializer/base.py new file mode 100644 index 0000000000..9141db8cea --- /dev/null +++ b/core/feed/serializer/base.py @@ -0,0 +1,32 @@ +from abc import ABC, abstractmethod +from typing import Generic, List, Optional, TypeVar + +from core.feed.types import FeedData, WorkEntryData +from core.util.opds_writer import OPDSMessage + +T = TypeVar("T") + + +class SerializerInterface(ABC, Generic[T]): + @classmethod + @abstractmethod + def to_string(cls, data: T) -> bytes: + ... + + @abstractmethod + def serialize_feed( + self, feed: FeedData, precomposed_entries: Optional[List[OPDSMessage]] = None + ) -> bytes: + ... + + @abstractmethod + def serialize_work_entry(self, entry: WorkEntryData) -> T: + ... + + @abstractmethod + def serialize_opds_message(self, message: OPDSMessage) -> T: + ... + + @abstractmethod + def content_type(self) -> str: + ... diff --git a/core/feed/serializer/opds.py b/core/feed/serializer/opds.py new file mode 100644 index 0000000000..d6ff9e49a2 --- /dev/null +++ b/core/feed/serializer/opds.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import datetime +from functools import partial +from typing import Any, Dict, List, Optional, cast + +from lxml import etree + +from core.feed.serializer.base import SerializerInterface +from core.feed.types import ( + Acquisition, + Author, + DataEntry, + FeedData, + FeedEntryType, + IndirectAcquisition, + WorkEntryData, +) +from core.util.datetime_helpers import utc_now +from core.util.opds_writer import OPDSFeed, OPDSMessage + +TAG_MAPPING = { + "indirectAcquisition": f"{{{OPDSFeed.OPDS_NS}}}indirectAcquisition", + "holds": f"{{{OPDSFeed.OPDS_NS}}}holds", + "copies": f"{{{OPDSFeed.OPDS_NS}}}copies", + "availability": f"{{{OPDSFeed.OPDS_NS}}}availability", + "licensor": f"{{{OPDSFeed.DRM_NS}}}licensor", + "patron": f"{{{OPDSFeed.SIMPLIFIED_NS}}}patron", + "series": f"{{{OPDSFeed.SCHEMA_NS}}}series", +} + +ATTRIBUTE_MAPPING = { + "vendor": f"{{{OPDSFeed.DRM_NS}}}vendor", + "scheme": f"{{{OPDSFeed.DRM_NS}}}scheme", + "username": f"{{{OPDSFeed.SIMPLIFIED_NS}}}username", + "authorizationIdentifier": f"{{{OPDSFeed.SIMPLIFIED_NS}}}authorizationIdentifier", + "rights": f"{{{OPDSFeed.DCTERMS_NS}}}rights", + "ProviderName": f"{{{OPDSFeed.BIBFRAME_NS}}}ProviderName", + "facetGroup": f"{{{OPDSFeed.OPDS_NS}}}facetGroup", + "activeFacet": f"{{{OPDSFeed.OPDS_NS}}}activeFacet", + "ratingValue": f"{{{OPDSFeed.SCHEMA_NS}}}ratingValue", +} + +AUTHOR_MAPPING = { + "name": f"{{{OPDSFeed.ATOM_NS}}}name", + "role": f"{{{OPDSFeed.OPF_NS}}}role", + "sort_name": f"{{{OPDSFeed.SIMPLIFIED_NS}}}sort_name", + "wikipedia_name": f"{{{OPDSFeed.SIMPLIFIED_NS}}}wikipedia_name", +} + + +class OPDS1Serializer(SerializerInterface[etree._Element], OPDSFeed): + """An OPDS 1.2 Atom feed serializer""" + + def __init__(self) -> None: + pass + + def _tag( + self, tag_name: str, *args: Any, mapping: Optional[Dict[str, str]] = None + ) -> etree._Element: + if not mapping: + mapping = TAG_MAPPING + return self.E(mapping.get(tag_name, tag_name), *args) + + def _attr_name( + self, attr_name: str, mapping: Optional[Dict[str, str]] = None + ) -> str: + if not mapping: + mapping = ATTRIBUTE_MAPPING + return mapping.get(attr_name, attr_name) + + def serialize_feed( + self, feed: FeedData, precomposed_entries: Optional[List[OPDSMessage]] = None + ) -> bytes: + # First we do metadata + serialized = self.E.feed() + + if feed.entrypoint: + serialized.set(f"{{{OPDSFeed.SIMPLIFIED_NS}}}entrypoint", feed.entrypoint) + + for name, metadata in feed.metadata.items(): + element = self._serialize_feed_entry(name, metadata) + serialized.append(element) + + for entry in feed.entries: + if entry.computed: + element = self.serialize_work_entry(entry.computed) + serialized.append(element) + + for data_entry in feed.data_entries: + element = self._serialize_data_entry(data_entry) + serialized.append(element) + + if precomposed_entries: + for precomposed in precomposed_entries: + if isinstance(precomposed, OPDSMessage): + serialized.append(self.serialize_opds_message(precomposed)) + + for link in feed.links: + serialized.append(self._serialize_feed_entry("link", link)) + + if feed.breadcrumbs: + breadcrumbs = OPDSFeed.E._makeelement( + f"{{{OPDSFeed.SIMPLIFIED_NS}}}breadcrumbs" + ) + for link in feed.breadcrumbs: + breadcrumbs.append(self._serialize_feed_entry("link", link)) + serialized.append(breadcrumbs) + + for link in feed.facet_links: + serialized.append(self._serialize_feed_entry("link", link)) + + etree.indent(serialized) + return self.to_string(serialized) + + def serialize_work_entry(self, feed_entry: WorkEntryData) -> etree._Element: + entry: etree._Element = OPDSFeed.entry() + + if feed_entry.additionalType: + entry.set( + f"{{{OPDSFeed.SCHEMA_NS}}}additionalType", feed_entry.additionalType + ) + + if feed_entry.title: + entry.append(OPDSFeed.E("title", feed_entry.title.text)) + if feed_entry.subtitle: + entry.append( + OPDSFeed.E( + f"{{{OPDSFeed.SCHEMA_NS}}}alternativeHeadline", + feed_entry.subtitle.text, + ) + ) + if feed_entry.summary: + entry.append(OPDSFeed.E("summary", feed_entry.summary.text)) + if feed_entry.pwid: + entry.append( + OPDSFeed.E(f"{{{OPDSFeed.SIMPLIFIED_NS}}}pwid", feed_entry.pwid) + ) + + if feed_entry.language: + entry.append( + OPDSFeed.E( + f"{{{OPDSFeed.DCTERMS_NS}}}language", feed_entry.language.text + ) + ) + if feed_entry.publisher: + entry.append( + OPDSFeed.E( + f"{{{OPDSFeed.DCTERMS_NS}}}publisher", feed_entry.publisher.text + ) + ) + if feed_entry.imprint: + entry.append( + OPDSFeed.E( + f"{{{OPDSFeed.BIB_SCHEMA_NS}}}publisherImprint", + feed_entry.imprint.text, + ) + ) + if feed_entry.issued: + # Entry.issued is the date the ebook came out, as distinct + # from Entry.published (which may refer to the print edition + # or some original edition way back when). + # + # For Dublin Core 'issued' we use Entry.issued if we have it + # and Entry.published if not. In general this means we use + # issued date for Gutenberg and published date for other + # sources. + # + # For the date the book was added to our collection we use + # atom:published. + # + # Note: feedparser conflates dc:issued and atom:published, so + # it can't be used to extract this information. However, these + # tags are consistent with the OPDS spec. + issued = feed_entry.issued + if isinstance(issued, datetime.datetime) or isinstance( + issued, datetime.date + ): + now = utc_now() + today = datetime.date.today() + issued_already = False + if isinstance(issued, datetime.datetime): + issued_already = issued <= now + elif isinstance(issued, datetime.date): + issued_already = issued <= today + if issued_already: + entry.append( + OPDSFeed.E( + f"{{{OPDSFeed.DCTERMS_NS}}}issued", + issued.isoformat().split("T")[0], + ) + ) + + if feed_entry.identifier: + entry.append(OPDSFeed.E("id", feed_entry.identifier)) + if feed_entry.distribution and ( + provider := getattr(feed_entry.distribution, "provider_name", None) + ): + entry.append( + OPDSFeed.E( + f"{{{OPDSFeed.BIBFRAME_NS}}}distribution", + **{f"{{{OPDSFeed.BIBFRAME_NS}}}ProviderName": provider}, + ) + ) + if feed_entry.published: + entry.append(OPDSFeed.E("published", feed_entry.published.text)) + if feed_entry.updated: + entry.append(OPDSFeed.E("updated", feed_entry.updated.text)) + + if feed_entry.series: + entry.append(self._serialize_series_entry(feed_entry.series)) + + for category in feed_entry.categories: + element = OPDSFeed.category( + scheme=category.scheme, term=category.term, label=category.label # type: ignore[attr-defined] + ) + entry.append(element) + + for rating in feed_entry.ratings: + rating_tag = self._serialize_feed_entry("Rating", rating) + entry.append(rating_tag) + + for author in feed_entry.authors: + entry.append(self._serialize_author_tag("author", author)) + for contributor in feed_entry.contributors: + entry.append(self._serialize_author_tag("contributor", contributor)) + + for link in feed_entry.image_links: + entry.append(OPDSFeed.link(**link.dict())) + + for link in feed_entry.acquisition_links: + element = self._serialize_acquistion_link(link) + entry.append(element) + + for link in feed_entry.other_links: + entry.append(OPDSFeed.link(**link.dict())) + + return entry + + def serialize_opds_message(self, entry: OPDSMessage) -> etree._Element: + return entry.tag + + def _serialize_series_entry(self, series: FeedEntryType) -> etree._Element: + entry = self._tag("series") + if name := getattr(series, "name", None): + entry.set("name", name) + if position := getattr(series, "position", None): + entry.append(self._tag("position", position)) + if link := getattr(series, "link", None): + entry.append(self._serialize_feed_entry("link", link)) + + return entry + + def _serialize_feed_entry( + self, tag: str, feed_entry: FeedEntryType + ) -> etree._Element: + """Serialize a feed entry type in a recursive and blind manner""" + entry: etree._Element = self._tag(tag) + for attrib, value in feed_entry: + if value is None: + continue + if isinstance(value, list): + for item in value: + entry.append(self._serialize_feed_entry(attrib, item)) + elif isinstance(value, FeedEntryType): + entry.append(self._serialize_feed_entry(attrib, value)) + else: + if attrib == "text": + entry.text = value + else: + entry.set( + ATTRIBUTE_MAPPING.get(attrib, attrib), + value if value is not None else "", + ) + return entry + + def _serialize_author_tag(self, tag: str, author: Author) -> etree._Element: + entry: etree._Element = self._tag(tag) + attr = partial(self._attr_name, mapping=AUTHOR_MAPPING) + _tag = partial(self._tag, mapping=AUTHOR_MAPPING) + if author.name: + element = _tag("name") + element.text = author.name + entry.append(element) + if author.role: + entry.set(attr("role"), author.role) + if author.link: + entry.append(self._serialize_feed_entry("link", author.link)) + + # Verbose + if author.sort_name: + entry.append(_tag("sort_name", author.sort_name)) + if author.wikipedia_name: + entry.append(_tag("wikipedia_name", author.wikipedia_name)) + if author.viaf: + entry.append(_tag("sameas", author.viaf)) + if author.lc: + entry.append(_tag("sameas", author.lc)) + return entry + + def _serialize_acquistion_link(self, link: Acquisition) -> etree._Element: + element = OPDSFeed.link(**link.link_attribs()) + + def _indirect(item: IndirectAcquisition) -> etree._Element: + tag = self._tag("indirectAcquisition") + tag.set("type", item.type) + for child in item.children: + tag.append(_indirect(child)) + return tag + + for indirect in link.indirect_acquisitions: + element.append(_indirect(indirect)) + + if link.availability_status: + avail_tag = self._tag("availability") + avail_tag.set("status", link.availability_status) + if link.availability_since: + avail_tag.set(self._attr_name("since"), link.availability_since) + if link.availability_until: + avail_tag.set(self._attr_name("until"), link.availability_until) + element.append(avail_tag) + + if link.holds_total is not None: + holds_tag = self._tag("holds") + holds_tag.set(self._attr_name("total"), link.holds_total) + if link.holds_position: + holds_tag.set(self._attr_name("position"), link.holds_position) + element.append(holds_tag) + + if link.copies_total is not None: + copies_tag = self._tag("copies") + copies_tag.set(self._attr_name("total"), link.copies_total) + if link.copies_available: + copies_tag.set(self._attr_name("available"), link.copies_available) + element.append(copies_tag) + + if link.lcp_hashed_passphrase: + element.append( + self._tag("hashed_passphrase", link.lcp_hashed_passphrase.text) + ) + + if link.drm_licensor: + element.append(self._serialize_feed_entry("licensor", link.drm_licensor)) + + return element + + def _serialize_data_entry(self, entry: DataEntry) -> etree._Element: + element = self._tag("entry") + if entry.title: + element.append(self._tag("title", entry.title)) + if entry.id: + element.append(self._tag("id", entry.id)) + for link in entry.links: + link_ele = self._serialize_feed_entry("link", link) + element.append(link_ele) + return element + + @classmethod + def to_string(cls, element: etree._Element) -> bytes: + return cast(bytes, etree.tostring(element)) + + def content_type(self) -> str: + return OPDSFeed.ACQUISITION_FEED_TYPE diff --git a/core/feed/serializer/opds2.py b/core/feed/serializer/opds2.py new file mode 100644 index 0000000000..26f59a3275 --- /dev/null +++ b/core/feed/serializer/opds2.py @@ -0,0 +1,213 @@ +import json +from collections import defaultdict +from typing import Any, Dict, List, Optional + +from core.feed.serializer.base import SerializerInterface +from core.feed.types import ( + Acquisition, + Author, + FeedData, + IndirectAcquisition, + Link, + WorkEntryData, +) +from core.model import Contributor +from core.util.opds_writer import OPDSMessage + +ALLOWED_ROLES = [ + "translator", + "editor", + "artist", + "illustrator", + "letterer", + "penciler", + "colorist", + "inker", + "narrator", +] +MARC_CODE_TO_ROLES = { + code: name.lower() + for name, code in Contributor.MARC_ROLE_CODES.items() + if name.lower() in ALLOWED_ROLES +} + + +class OPDS2Serializer(SerializerInterface[Dict[str, Any]]): + def __init__(self) -> None: + pass + + def serialize_feed( + self, feed: FeedData, precomposed_entries: Optional[List[Any]] = None + ) -> bytes: + serialized: Dict[str, Any] = {"publications": []} + serialized["metadata"] = self._serialize_metadata(feed) + + for entry in feed.entries: + if entry.computed: + publication = self.serialize_work_entry(entry.computed) + serialized["publications"].append(publication) + + serialized.update(self._serialize_feed_links(feed)) + + return self.to_string(serialized) + + def _serialize_metadata(self, feed: FeedData) -> Dict[str, Any]: + fmeta = feed.metadata + metadata: Dict[str, Any] = {} + if title := fmeta.get("title"): + metadata["title"] = title.text + if item_count := fmeta.get("items_per_page"): + metadata["itemsPerPage"] = int(item_count.text or 0) + return metadata + + def serialize_opds_message(self, entry: OPDSMessage) -> Dict[str, Any]: + return dict(urn=entry.urn, description=entry.message) + + def serialize_work_entry(self, data: WorkEntryData) -> Dict[str, Any]: + metadata: Dict[str, Any] = {} + if data.additionalType: + metadata["@type"] = data.additionalType + + if data.title: + metadata["title"] = data.title.text + if data.sort_title: + metadata["sortAs"] = data.sort_title.text + + if data.subtitle: + metadata["subtitle"] = data.subtitle.text + if data.identifier: + metadata["identifier"] = data.identifier + if data.language: + metadata["language"] = data.language.text + if data.updated: + metadata["modified"] = data.updated.text + if data.published: + metadata["published"] = data.published.text + if data.summary: + metadata["description"] = data.summary.text + + if data.publisher: + metadata["publisher"] = dict(name=data.publisher.text) + if data.imprint: + metadata["imprint"] = dict(name=data.imprint.text) + + subjects = [] + if data.categories: + for subject in data.categories: + subjects.append( + { + "scheme": subject.scheme, # type: ignore[attr-defined] + "name": subject.label, # type: ignore[attr-defined] + "sortAs": subject.label, # type: ignore[attr-defined] # Same as above, don't think we have an alternate + } + ) + metadata["subject"] = subjects + + if data.series: + name = getattr(data.series, "name", None) + position = int(getattr(data.series, "position", 1)) + if name: + metadata["belongsTo"] = dict(name=name, position=position) + + if len(data.authors): + metadata["author"] = self._serialize_contributor(data.authors[0]) + for contributor in data.contributors: + if role := MARC_CODE_TO_ROLES.get(contributor.role or "", None): + metadata[role] = self._serialize_contributor(contributor) + + images = [self._serialize_link(link) for link in data.image_links] + links = [self._serialize_link(link) for link in data.other_links] + + for acquisition in data.acquisition_links: + links.append(self._serialize_acquisition_link(acquisition)) + + publication = {"metadata": metadata, "links": links, "images": images} + return publication + + def _serialize_link(self, link: Link) -> Dict[str, Any]: + serialized = {"href": link.href, "rel": link.rel} + if link.type: + serialized["type"] = link.type + if link.title: + serialized["title"] = link.title + return serialized + + def _serialize_acquisition_link(self, link: Acquisition) -> Dict[str, Any]: + item = self._serialize_link(link) + + def _indirect(indirect: IndirectAcquisition) -> Dict[str, Any]: + result: Dict[str, Any] = dict(type=indirect.type) + if indirect.children: + result["child"] = [] + for child in indirect.children: + result["child"].append(_indirect(child)) + return result + + props: Dict[str, Any] = {} + if link.availability_status: + state = link.availability_status + if link.is_loan: + state = "ready" + elif link.is_hold: + state = "reserved" + # This only exists in the serializer because there is no case where cancellable is false, + # that logic should be in the annotator if it ever occurs + props["actions"] = dict(cancellable=True) + props["availability"] = dict(state=state) + if link.availability_since: + props["availability"]["since"] = link.availability_since + if link.availability_until: + props["availability"]["until"] = link.availability_until + + if link.indirect_acquisitions: + props["indirectAcquisition"] = [] + for indirect in link.indirect_acquisitions: + props["indirectAcquisition"].append(_indirect(indirect)) + + if link.lcp_hashed_passphrase: + props["lcp_hashed_passphrase"] = link.lcp_hashed_passphrase + + if link.drm_licensor: + props["licensor"] = { + "clientToken": getattr(link.drm_licensor, "clientToken"), + "vendor": getattr(link.drm_licensor, "vendor"), + } + + if props: + item["properties"] = props + + return item + + def _serialize_feed_links(self, feed: FeedData) -> Dict[str, Any]: + link_data: Dict[str, List[Dict[str, Any]]] = {"links": [], "facets": []} + for link in feed.links: + link_data["links"].append(self._serialize_link(link)) + + facet_links: Dict[str, Any] = defaultdict(lambda: {"metadata": {}, "links": []}) + for link in feed.facet_links: + group = getattr(link, "facetGroup", None) + if group: + facet_links[group]["links"].append(self._serialize_link(link)) + facet_links[group]["metadata"]["title"] = group + for _, facets in facet_links.items(): + link_data["facets"].append(facets) + + return link_data + + def _serialize_contributor(self, author: Author) -> Dict[str, Any]: + result: Dict[str, Any] = {"name": author.name} + if author.sort_name: + result["sortAs"] = author.sort_name + if author.link: + link = self._serialize_link(author.link) + # OPDS2 does not need "title" in the link + link.pop("title", None) + result["links"] = [link] + return result + + def content_type(self) -> str: + return "application/opds+json" + + @classmethod + def to_string(cls, data: Dict[str, Any]) -> bytes: + return json.dumps(data, indent=2).encode() diff --git a/core/feed/types.py b/core/feed/types.py new file mode 100644 index 0000000000..1b6d1d4000 --- /dev/null +++ b/core/feed/types.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date, datetime +from typing import Any, Dict, Generator, List, Optional, Tuple, cast + +from typing_extensions import Self + +from core.model import LicensePool, Work +from core.model.edition import Edition +from core.model.identifier import Identifier + +NO_SUCH_KEY = object() + + +@dataclass +class BaseModel: + def _vars(self) -> Generator[Tuple[str, Any], None, None]: + """Yield attributes as a tuple""" + _attrs = vars(self) + for name, value in _attrs.items(): + if name.startswith("_"): + continue + elif callable(value): + continue + yield name, value + + def dict(self) -> Dict[str, Any]: + """Dataclasses do not return undefined attributes via `asdict` so we must implement this ourselves""" + attrs = {} + for name, value in self: + if isinstance(value, BaseModel): + attrs[name] = value.dict() + else: + attrs[name] = value + return attrs + + def __iter__(self) -> Generator[Tuple[str, Any], None, None]: + """Allow attribute iteration""" + yield from self._vars() + + def get(self, name: str, *default: Any) -> Any: + """Convenience function. Mimics getattr""" + value = getattr(self, name, NO_SUCH_KEY) + if value is NO_SUCH_KEY: + if len(default) > 0: + return default[0] + else: + raise AttributeError(f"No attribute '{name}' found in object {self}") + return value + + +@dataclass +class FeedEntryType(BaseModel): + text: Optional[str] = None + + @classmethod + def create(cls, **kwargs: Any) -> Self: + """Create a new object with arbitrary data""" + obj = cls() + obj.add_attributes(kwargs) + return obj + + def add_attributes(self, attrs: Dict[str, Any]) -> None: + for name, data in attrs.items(): + setattr(self, name, data) + + def children(self) -> Generator[Tuple[str, FeedEntryType], None, None]: + """Yield all FeedEntryType attributes""" + for name, value in self: + if isinstance(value, self.__class__): + yield name, value + return + + +@dataclass +class Link(FeedEntryType): + href: Optional[str] = None + rel: Optional[str] = None + type: Optional[str] = None + + # Additional types + role: Optional[str] = None + title: Optional[str] = None + + def dict(self) -> Dict[str, Any]: + """A dict without None values""" + d = super().dict() + santized = {} + for k, v in d.items(): + if v is not None: + santized[k] = v + return santized + + def link_attribs(self) -> Dict[str, Any]: + d = dict(href=self.href) + for key in ["rel", "type"]: + if (value := getattr(self, key, None)) is not None: + d[key] = value + return d + + +@dataclass +class IndirectAcquisition(BaseModel): + type: Optional[str] = None + children: List[IndirectAcquisition] = field(default_factory=list) + + +@dataclass +class Acquisition(Link): + holds_position: Optional[str] = None + holds_total: Optional[str] = None + + copies_available: Optional[str] = None + copies_total: Optional[str] = None + + availability_status: Optional[str] = None + availability_since: Optional[str] = None + availability_until: Optional[str] = None + + rights: Optional[str] = None + + lcp_hashed_passphrase: Optional[FeedEntryType] = None + drm_licensor: Optional[FeedEntryType] = None + + indirect_acquisitions: List[IndirectAcquisition] = field(default_factory=list) + + # Signal if the acquisition is for a loan or a hold for the patron + is_loan: bool = False + is_hold: bool = False + + +@dataclass +class Author(FeedEntryType): + name: Optional[str] = None + sort_name: Optional[str] = None + viaf: Optional[str] = None + role: Optional[str] = None + family_name: Optional[str] = None + wikipedia_name: Optional[str] = None + lc: Optional[str] = None + link: Optional[Link] = None + + +@dataclass +class WorkEntryData(BaseModel): + """All the metadata possible for a work. This is not a FeedEntryType because we want strict control.""" + + additionalType: Optional[str] = None + identifier: Optional[str] = None + pwid: Optional[str] = None + issued: Optional[datetime | date] = None + + summary: Optional[FeedEntryType] = None + language: Optional[FeedEntryType] = None + publisher: Optional[FeedEntryType] = None + published: Optional[FeedEntryType] = None + updated: Optional[FeedEntryType] = None + title: Optional[FeedEntryType] = None + sort_title: Optional[FeedEntryType] = None + subtitle: Optional[FeedEntryType] = None + series: Optional[FeedEntryType] = None + imprint: Optional[FeedEntryType] = None + + authors: List[Author] = field(default_factory=list) + contributors: List[Author] = field(default_factory=list) + categories: List[FeedEntryType] = field(default_factory=list) + ratings: List[FeedEntryType] = field(default_factory=list) + distribution: Optional[FeedEntryType] = None + + # Links + acquisition_links: List[Acquisition] = field(default_factory=list) + image_links: List[Link] = field(default_factory=list) + other_links: List[Link] = field(default_factory=list) + + +@dataclass +class WorkEntry(BaseModel): + work: Work + edition: Edition + identifier: Identifier + license_pool: Optional[LicensePool] = None + + # Actual, computed feed data + computed: Optional[WorkEntryData] = None + + def __init__( + self, + work: Optional[Work] = None, + edition: Optional[Edition] = None, + identifier: Optional[Identifier] = None, + license_pool: Optional[LicensePool] = None, + ) -> None: + if None in (work, edition, identifier): + raise ValueError( + "Work, Edition or Identifier cannot be None while initializing an entry" + ) + self.work = cast(Work, work) + self.edition = cast(Edition, edition) + self.identifier = cast(Identifier, identifier) + self.license_pool = license_pool + + +class DataEntryTypes: + NAVIGATION = "navigation" + + +@dataclass +class DataEntry(FeedEntryType): + """Other kinds of information, like entries of a navigation feed""" + + type: Optional[str] = None + title: Optional[str] = None + id: Optional[str] = None + links: List[Link] = field(default_factory=list) + + +@dataclass +class FeedData(BaseModel): + links: List[Link] = field(default_factory=list) + breadcrumbs: List[Link] = field(default_factory=list) + facet_links: List[Link] = field(default_factory=list) + entries: List[WorkEntry] = field(default_factory=list) + data_entries: List[DataEntry] = field(default_factory=list) + metadata: Dict[str, FeedEntryType] = field(default_factory=dict) + entrypoint: Optional[str] = None + + class Config: + arbitrary_types_allowed = True + + def add_link(self, href: str, **kwargs: Any) -> None: + self.links.append(Link(href=href, **kwargs)) + + def add_metadata( + self, name: str, feed_entry: Optional[FeedEntryType] = None, **kwargs: Any + ) -> None: + if not feed_entry: + self.metadata[name] = FeedEntryType(**kwargs) + else: + self.metadata[name] = feed_entry diff --git a/core/feed/util.py b/core/feed/util.py new file mode 100644 index 0000000000..5519f0a5b5 --- /dev/null +++ b/core/feed/util.py @@ -0,0 +1,27 @@ +import datetime +from typing import Union + +import pytz + +TIME_FORMAT_UTC = "%Y-%m-%dT%H:%M:%S+00:00" +TIME_FORMAT_NAIVE = "%Y-%m-%dT%H:%M:%SZ" + + +def strftime(date: Union[datetime.datetime, datetime.date]) -> str: + """ + Format a date for the OPDS feeds. + + 'A Date construct is an element whose content MUST conform to the + "date-time" production in [RFC3339]. In addition, an uppercase "T" + character MUST be used to separate date and time, and an uppercase + "Z" character MUST be present in the absence of a numeric time zone + offset.' (https://tools.ietf.org/html/rfc4287#section-3.3) + """ + if isinstance(date, datetime.datetime) and date.tzinfo is not None: + # Convert to UTC to make the formatting easier. + fmt = TIME_FORMAT_UTC + date = date.astimezone(pytz.UTC) + else: + fmt = TIME_FORMAT_NAIVE + + return date.strftime(fmt) diff --git a/core/model/constants.py b/core/model/constants.py index 230b7f3944..66052d9a55 100644 --- a/core/model/constants.py +++ b/core/model/constants.py @@ -230,6 +230,7 @@ class LinkRelations: SHORT_DESCRIPTION = "http://librarysimplified.org/terms/rel/short-description" AUTHOR = "http://schema.org/author" ALTERNATE = "alternate" + FACET_REL = "http://opds-spec.org/facet" # The rel for a link we feed to clients for samples/previews. CLIENT_SAMPLE = "preview" diff --git a/core/model/licensing.py b/core/model/licensing.py index 788fe095af..93ab351e59 100644 --- a/core/model/licensing.py +++ b/core/model/licensing.py @@ -255,9 +255,9 @@ class LicensePool(Base): open_access = Column(Boolean, index=True) last_checked = Column(DateTime(timezone=True), index=True) - licenses_owned = Column(Integer, default=0, index=True) - licenses_available = Column(Integer, default=0, index=True) - licenses_reserved = Column(Integer, default=0) + licenses_owned: int = Column(Integer, default=0, index=True) + licenses_available: int = Column(Integer, default=0, index=True) + licenses_reserved: int = Column(Integer, default=0) patrons_in_hold_queue = Column(Integer, default=0) # This lets us cache the work of figuring out the best open access diff --git a/core/model/work.py b/core/model/work.py index 3dd8f8edef..063f7cd446 100644 --- a/core/model/work.py +++ b/core/model/work.py @@ -7,7 +7,7 @@ from collections import Counter from datetime import date, datetime from decimal import Decimal -from typing import TYPE_CHECKING, Any, List, Union, cast +from typing import TYPE_CHECKING, Any, List, Optional, Union, cast import pytz from sqlalchemy import ( @@ -287,7 +287,7 @@ def sort_author(self): return self.presentation_edition.sort_author or self.presentation_edition.author @property - def language(self): + def language(self) -> Optional[str]: if self.presentation_edition: return self.presentation_edition.language return None diff --git a/core/opds_schema.py b/core/opds_schema.py index 23f24a0b87..05717c1811 100644 --- a/core/opds_schema.py +++ b/core/opds_schema.py @@ -36,6 +36,8 @@ def validate_schema(self, schema_path: str, feed: dict): class OPDS2SchemaValidation(OPDS2ImportMonitor, OPDS2SchemaValidationMixin): def import_one_feed(self, feed): + if type(feed) in (str, bytes): + feed = json.loads(feed) self.validate_schema("core/resources/opds2_schema/feed.schema.json", feed) return [], [] diff --git a/pyproject.toml b/pyproject.toml index 83cb0192f1..0f7a967861 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ module = [ "api.circulation", "api.discovery.*", "api.integration.*", + "core.feed.*", "core.integration.*", "core.model.announcements", "core.model.hassessioncache", diff --git a/scripts.py b/scripts.py index f088686dfb..2d3888e05f 100644 --- a/scripts.py +++ b/scripts.py @@ -32,6 +32,7 @@ from api.overdrive import OverdriveAPI from core.entrypoint import EntryPoint from core.external_search import ExternalSearchIndex +from core.feed.acquisition import OPDSAcquisitionFeed from core.lane import Facets, FeaturedFacets, Lane, Pagination from core.log import LogConfiguration from core.marc import MARCExporter @@ -55,7 +56,6 @@ pg_advisory_lock, ) from core.model.configuration import ExternalIntegrationLink -from core.opds import AcquisitionFeed from core.scripts import ( IdentifierInputScript, LaneSweeperScript, @@ -483,17 +483,17 @@ def do_generate(self, lane, facets, pagination, feed_class=None): library = lane.get_library(self._db) annotator = self.app.manager.annotator(lane, facets=facets) url = annotator.feed_url(lane, facets=facets, pagination=pagination) - feed_class = feed_class or AcquisitionFeed + feed_class = feed_class or OPDSAcquisitionFeed return feed_class.page( _db=self._db, title=title, url=url, worklist=lane, annotator=annotator, - facets=facets, pagination=pagination, - max_age=0, - ) + facets=facets, + search_engine=None, + ).as_response(max_age=0) class CacheOPDSGroupFeedPerLane(CacheRepresentationPerLane): @@ -512,7 +512,7 @@ def do_generate(self, lane, facets, pagination, feed_class=None): title = lane.display_name annotator = self.app.manager.annotator(lane, facets=facets) url = annotator.groups_url(lane, facets) - feed_class = feed_class or AcquisitionFeed + feed_class = feed_class or OPDSAcquisitionFeed # Since grouped feeds are only cached for lanes that have sublanes, # there's no need to consider the case of a lane with no sublanes, @@ -523,9 +523,10 @@ def do_generate(self, lane, facets, pagination, feed_class=None): url=url, worklist=lane, annotator=annotator, - max_age=0, + pagination=None, facets=facets, - ) + search_engine=None, + ).as_response(max_age=0) def facets(self, lane): """Generate a Facets object for each of the library's enabled diff --git a/tests/api/feed/equivalence/test_feed_equivalence.py b/tests/api/feed/equivalence/test_feed_equivalence.py new file mode 100644 index 0000000000..cf07fd153d --- /dev/null +++ b/tests/api/feed/equivalence/test_feed_equivalence.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +from lxml import etree + +from api.admin.opds import AdminAnnotator as OldAdminAnnotator +from api.admin.opds import AdminFeed as OldAdminFeed +from api.app import app +from api.opds import LibraryAnnotator as OldLibraryAnnotator +from api.opds import LibraryLoanAndHoldAnnotator as OldLibraryLoanAndHoldAnnotator +from core.external_search import MockExternalSearchIndex +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.admin import AdminFeed +from core.feed.annotator.admin import AdminAnnotator +from core.feed.annotator.circulation import LibraryAnnotator +from core.feed.navigation import NavigationFeed +from core.lane import Facets, Pagination +from core.model.work import Work +from core.opds import AcquisitionFeed +from core.opds import NavigationFeed as OldNavigationFeed +from tests.api.feed.test_library_annotator import ( # noqa + LibraryAnnotatorFixture, + annotator_fixture, + patch_url_for, +) +from tests.fixtures.database import DatabaseTransactionFixture + + +def format_tags(tags1, tags2): + result = "" + result += "TAG1\n" + for tag in tags1: + result += f"{tag[1:]}\n" + result += "TAG2\n" + for tag in tags2: + result += f"{tag[1:]}\n" + return result + + +def assert_equal_xmls(xml1: str | etree._Element, xml2: str | etree._Element): + if isinstance(xml1, str) or isinstance(xml1, bytes): + parsed1 = etree.fromstring(xml1) + else: + parsed1 = xml1 + + if isinstance(xml2, str) or isinstance(xml2, bytes): + parsed2 = etree.fromstring(xml2) + else: + parsed2 = xml2 + + # Pull out comparable information + tags1 = [(tag, tag.tag, tag.text, tag.attrib) for tag in parsed1[1:]] + tags2 = [(tag, tag.tag, tag.text, tag.attrib) for tag in parsed2[1:]] + # Sort the tags on the information so it's easy to compare sequentially + tags1.sort(key=lambda x: (x[1], x[2] or "", x[3].values())) + tags2.sort(key=lambda x: (x[1], x[2] or "", x[3].values())) + + assert len(tags1) == len(tags2), format_tags(tags1, tags2) + + # Assert every tag is equal + for ix, tag1 in enumerate(tags1): + tag2 = tags2[ix] + # Comparable information should be equivalent + if tag1[1:] == tag2[1:]: + assert_equal_xmls(tag1[0], tag2[0]) + break + else: + assert False, format_tags([tag1], tags2) + + +class TestFeedEquivalence: + def test_page_feed(self, annotator_fixture: LibraryAnnotatorFixture): + db = annotator_fixture.db + lane = annotator_fixture.lane + library = db.default_library() + + work1 = db.work(with_license_pool=True) + work2 = db.work(with_open_access_download=True) + + search_index = MockExternalSearchIndex() + search_index.bulk_update([work1, work2]) + + with app.test_request_context("/"): + new_annotator = LibraryAnnotator(None, lane, library) + new_feed = OPDSAcquisitionFeed.page( + db.session, + lane.display_name, + "http://test-url/", + lane, + new_annotator, + Facets.default(library), + Pagination.default(), + search_index, + ) + + old_annotator = OldLibraryAnnotator(None, lane, library) + old_feed = AcquisitionFeed.page( + db.session, + lane.display_name, + "http://test-url/", + lane, + old_annotator, + Facets.default(library), + Pagination.default(), + search_engine=search_index, + ) + + assert_equal_xmls(str(old_feed), new_feed.serialize()) + + def test_page_feed_with_loan_annotator( + self, annotator_fixture: LibraryAnnotatorFixture + ): + db = annotator_fixture.db + library = db.default_library() + work1 = db.work(with_license_pool=True) + patron = db.patron() + work1.active_license_pool(library).loan_to(patron) + + with app.test_request_context("/"): + new_feed = OPDSAcquisitionFeed.active_loans_for(None, patron).as_response() + old_feed = OldLibraryLoanAndHoldAnnotator.active_loans_for(None, patron) + + assert_equal_xmls(str(old_feed), str(new_feed)) + + def test_groups_feed(self, annotator_fixture: LibraryAnnotatorFixture): + db = annotator_fixture.db + lane = annotator_fixture.lane + de_lane = db.lane(parent=lane, languages=["de"]) + library = db.default_library() + + work1 = db.work(with_license_pool=True) + work2 = db.work(with_open_access_download=True, language="de") + + search_index = MockExternalSearchIndex() + search_index.bulk_update([work1, work2]) + + patron = db.patron() + work1.active_license_pool(library).loan_to(patron) + + with app.test_request_context("/"): + new_annotator = LibraryAnnotator(None, lane, library) + new_feed = OPDSAcquisitionFeed.groups( + db.session, + "Groups", + "http://groups/", + lane, + new_annotator, + Pagination.default(), + Facets.default(library), + search_index, + ) + + old_annotator = OldLibraryAnnotator(None, lane, library) + old_feed = AcquisitionFeed.groups( + db.session, + "Groups", + "http://groups/", + lane, + old_annotator, + pagination=Pagination.default(), + facets=Facets.default(library), + search_engine=search_index, + ) + + assert_equal_xmls(str(old_feed), new_feed.serialize().decode()) + + def test_search_feed(self, annotator_fixture: LibraryAnnotatorFixture): + db = annotator_fixture.db + lane = annotator_fixture.lane + de_lane = db.lane(parent=lane, languages=["de"]) + library = db.default_library() + + work1 = db.work(with_license_pool=True) + work2 = db.work(with_open_access_download=True, language="de") + + search_index = MockExternalSearchIndex() + search_index.bulk_update([work1, work2]) + + patron = db.patron() + work1.active_license_pool(library).loan_to(patron) + + with app.test_request_context("/"): + new_annotator = LibraryAnnotator(None, lane, library) + new_feed = OPDSAcquisitionFeed.search( # type: ignore[union-attr] + db.session, + "Search", + "http://search/", + lane, + search_index, + "query", + new_annotator, + Pagination.default(), + Facets.default(library), + ).as_response() + + old_annotator = OldLibraryAnnotator(None, lane, library) + old_feed = AcquisitionFeed.search( + db.session, + "Search", + "http://search/", + lane, + search_index, + "query", + Pagination.default(), + Facets.default(library), + old_annotator, + ) + + assert_equal_xmls(str(old_feed), str(new_feed)) + + def test_from_query_feed(self, annotator_fixture: LibraryAnnotatorFixture): + db = annotator_fixture.db + lane = annotator_fixture.lane + de_lane = db.lane(parent=lane, languages=["de"]) + library = db.default_library() + + work1 = db.work(with_license_pool=True) + work2 = db.work(with_open_access_download=True, language="de") + + search_index = MockExternalSearchIndex() + search_index.bulk_update([work1, work2]) + + patron = db.patron() + work1.active_license_pool(library).loan_to(patron) + + def url_fn(page): + return f"http://pagination?page={page}" + + query = db.session.query(Work) + + with app.test_request_context("/"): + new_annotator = LibraryAnnotator(None, lane, library) + new_feed = OPDSAcquisitionFeed.from_query( + query, + db.session, + "Search", + "http://search/", + Pagination(), + url_fn, + new_annotator, + ) + + old_annotator = OldLibraryAnnotator(None, lane, library) + old_feed = AcquisitionFeed.from_query( + query, + db.session, + "Search", + "http://search/", + Pagination(), + url_fn, + old_annotator, + ) + + assert_equal_xmls(str(old_feed), new_feed.serialize()) + + +class TestAdminAnnotator: + def test_suppressed(self, annotator_fixture: LibraryAnnotatorFixture): + db = annotator_fixture.db + library = db.default_library() + + work1 = db.work(with_open_access_download=True) + pool = work1.active_license_pool() + pool.suppressed = True + + with app.test_request_context("/"): + new_annotator = AdminAnnotator(None, library) + new_feed = AdminFeed.suppressed( + db.session, "", "http://verbose", new_annotator + ) + + old_annotator = OldAdminAnnotator(None, library) + old_feed = OldAdminFeed.suppressed( + db.session, "", "http://verbose", old_annotator + ) + + assert_equal_xmls(str(old_feed), new_feed.serialize()) + + +class TestNavigationFeed: + def test_feed(self, db: DatabaseTransactionFixture): + lane = db.lane() + child1 = db.lane(parent=lane) + child2 = db.lane(parent=lane) + + with app.test_request_context("/"): + new_annotator = LibraryAnnotator(None, lane, db.default_library()) + new_feed = NavigationFeed.navigation( + db.session, "Navigation", "http://navigation", lane, new_annotator + ) + + old_annotator = OldLibraryAnnotator(None, lane, db.default_library()) + old_feed = OldNavigationFeed.navigation( + db.session, "Navigation", "http://navigation", lane, old_annotator + ) + + assert_equal_xmls(str(old_feed), str(new_feed.as_response())) diff --git a/tests/api/feed/fixtures.py b/tests/api/feed/fixtures.py new file mode 100644 index 0000000000..002d806bac --- /dev/null +++ b/tests/api/feed/fixtures.py @@ -0,0 +1,47 @@ +import urllib +from dataclasses import dataclass +from functools import partial +from typing import Any, Callable +from unittest.mock import patch + +import pytest +from flask import has_request_context + +from core.feed.annotator.circulation import CirculationManagerAnnotator + + +def _patched_url_for(*args: Any, _original=None, **kwargs: Any) -> str: + """Test mode url_for for the annotators + :param _original: Is the original Annotator.url_for method + """ + if has_request_context() and _original: + # Ignore the patch if we have a request context + return _original(object(), *args, **kwargs) + # Generate a plausible-looking URL that doesn't depend on Flask + # being set up. + host = "host" + url = ("http://%s/" % host) + "/".join(args) + connector = "?" + for k, v in sorted(kwargs.items()): + if v is None: + v = "" + v = urllib.parse.quote(str(v)) + k = urllib.parse.quote(str(k)) + url += connector + f"{k}={v}" + connector = "&" + return url + + +@dataclass +class PatchedUrlFor: + patched_url_for: Callable + + +@pytest.fixture(scope="function") +def patch_url_for(): + """Patch the url_for method for annotators""" + with patch( + "core.feed.annotator.circulation.CirculationManagerAnnotator.url_for", + new=partial(_patched_url_for, _original=CirculationManagerAnnotator.url_for), + ) as patched: + yield PatchedUrlFor(patched) diff --git a/tests/api/feed/test_admin.py b/tests/api/feed/test_admin.py new file mode 100644 index 0000000000..283823e337 --- /dev/null +++ b/tests/api/feed/test_admin.py @@ -0,0 +1,285 @@ +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.admin import AdminFeed +from core.feed.annotator.admin import AdminAnnotator +from core.feed.types import FeedData +from core.lane import Pagination +from core.model.configuration import ExternalIntegration, ExternalIntegrationLink +from core.model.datasource import DataSource +from core.model.measurement import Measurement +from tests.api.feed.fixtures import PatchedUrlFor, patch_url_for # noqa +from tests.fixtures.database import DatabaseTransactionFixture + + +class TestOPDS: + def links(self, feed: FeedData, rel=None): + all_links = feed.links + feed.facet_links + feed.breadcrumbs + links = sorted(all_links, key=lambda x: (x.rel, getattr(x, "title", None))) + r = [] + for l in links: + if not rel or l.rel == rel or (isinstance(rel, list) and l.rel in rel): + r.append(l) + return r + + def test_feed_includes_staff_rating( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + work = db.work(with_open_access_download=True) + lp = work.license_pools[0] + staff_data_source = DataSource.lookup(db.session, DataSource.LIBRARY_STAFF) + lp.identifier.add_measurement( + staff_data_source, Measurement.RATING, 3, weight=1000 + ) + + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, db.default_library()), + ) + feed.generate_feed() + + [entry] = feed._feed.entries + assert entry.computed is not None + assert len(entry.computed.ratings) == 2 + assert 3 == float(entry.computed.ratings[1].ratingValue) # type: ignore[attr-defined] + assert Measurement.RATING == entry.computed.ratings[1].additionalType # type: ignore[attr-defined] + + def test_feed_includes_refresh_link( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + work = db.work(with_open_access_download=True) + lp = work.license_pools[0] + lp.suppressed = False + db.session.commit() + + # If the metadata wrangler isn't configured, the link is left out. + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, db.default_library()), + ) + [entry] = feed._feed.entries + assert entry.computed is not None + assert [] == [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/refresh" + ] + + def test_feed_includes_suppress_link( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + work = db.work(with_open_access_download=True) + lp = work.license_pools[0] + lp.suppressed = False + db.session.commit() + + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, db.default_library()), + ) + [entry] = feed._feed.entries + assert entry.computed is not None + [suppress_link] = [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/hide" + ] + assert suppress_link.href and lp.identifier.identifier in suppress_link.href + unsuppress_links = [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/restore" + ] + assert 0 == len(unsuppress_links) + + lp.suppressed = True + db.session.commit() + + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, db.default_library()), + ) + [entry] = feed._feed.entries + assert entry.computed is not None + [unsuppress_link] = [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/restore" + ] + assert unsuppress_link.href and lp.identifier.identifier in unsuppress_link.href + suppress_links = [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/hide" + ] + assert 0 == len(suppress_links) + + def test_feed_includes_edit_link( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + work = db.work(with_open_access_download=True) + lp = work.license_pools[0] + + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, db.default_library()), + ) + [entry] = feed._feed.entries + assert entry.computed is not None + [edit_link] = [x for x in entry.computed.other_links if x.rel == "edit"] + assert edit_link.href and lp.identifier.identifier in edit_link.href + + def test_feed_includes_change_cover_link( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + work = db.work(with_open_access_download=True) + lp = work.license_pools[0] + library = db.default_library() + + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, library), + ) + [entry] = feed._feed.entries + assert entry.computed is not None + + # Since there's no storage integration, the change cover link isn't included. + assert [] == [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/change_cover" + ] + + # There is now a covers storage integration that is linked to the external + # integration for a collection that the work is in. It will use that + # covers mirror and the change cover link is included. + storage = db.external_integration( + ExternalIntegration.S3, ExternalIntegration.STORAGE_GOAL + ) + storage.username = "user" + storage.password = "pass" + + collection = db.collection() + purpose = ExternalIntegrationLink.COVERS + external_integration_link = db.external_integration_link( + integration=collection._external_integration, + other_integration=storage, + purpose=purpose, + ) + library.collections.append(collection) + work = db.work(with_open_access_download=True, collection=collection) + lp = work.license_pools[0] + feed = OPDSAcquisitionFeed( + "test", + "url", + [work], + AdminAnnotator(None, library), + ) + [entry] = feed._feed.entries + assert entry.computed is not None + + [change_cover_link] = [ + x + for x in entry.computed.other_links + if x.rel == "http://librarysimplified.org/terms/rel/change_cover" + ] + assert ( + change_cover_link.href + and lp.identifier.identifier in change_cover_link.href + ) + + def test_suppressed_feed( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + # Test the ability to show a paginated feed of suppressed works. + + work1 = db.work(with_open_access_download=True) + work1.license_pools[0].suppressed = True + + work2 = db.work(with_open_access_download=True) + work2.license_pools[0].suppressed = True + + # This work won't be included in the feed since its + # suppressed pool is superceded. + work3 = db.work(with_open_access_download=True) + work3.license_pools[0].suppressed = True + work3.license_pools[0].superceded = True + + pagination = Pagination(size=1) + annotator = MockAnnotator(db.default_library()) + titles = [work1.title, work2.title] + + def make_page(pagination): + return AdminFeed.suppressed( + _db=db.session, + title="Hidden works", + url=db.fresh_url(), + annotator=annotator, + pagination=pagination, + ) + + feed = make_page(pagination)._feed + assert 1 == len(feed.entries) + assert feed.entries[0].computed.title.text in titles + titles.remove(feed.entries[0].computed.title.text) + [remaining_title] = titles + + # Make sure the links are in place. + [start] = self.links(feed, "start") + assert annotator.groups_url(None) == start.href + assert annotator.top_level_title() == start.title + + [up] = self.links(feed, "up") + assert annotator.groups_url(None) == up.href + assert annotator.top_level_title() == up.title + + [next_link] = self.links(feed, "next") + assert annotator.suppressed_url(pagination.next_page) == next_link.href + + # This was the first page, so no previous link. + assert [] == self.links(feed, "previous") + + # Now get the second page and make sure it has a 'previous' link. + second_page = make_page(pagination.next_page)._feed + [previous] = self.links(second_page, "previous") + assert annotator.suppressed_url(pagination) == previous.href + assert 1 == len(second_page.entries) + assert remaining_title == second_page.entries[0].computed.title.text + + # The third page is empty. + third_page = make_page(pagination.next_page.next_page)._feed + [previous] = self.links(third_page, "previous") + assert annotator.suppressed_url(pagination.next_page) == previous.href + assert 0 == len(third_page.entries) + + +class MockAnnotator(AdminAnnotator): + def __init__(self, library): + super().__init__(None, library) + + def groups_url(self, lane): + if lane: + name = lane.name + else: + name = "" + return "http://groups/%s" % name + + def suppressed_url(self, pagination): + base = "http://suppressed/" + sep = "?" + if pagination: + base += sep + pagination.query_string + return base + + def annotate_feed(self, feed): + super().annotate_feed(feed) diff --git a/tests/api/feed/test_annotators.py b/tests/api/feed/test_annotators.py new file mode 100644 index 0000000000..e4e3b1032c --- /dev/null +++ b/tests/api/feed/test_annotators.py @@ -0,0 +1,469 @@ +from datetime import timedelta + +from core.classifier import Classifier +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.base import Annotator +from core.feed.annotator.circulation import CirculationManagerAnnotator +from core.feed.annotator.verbose import VerboseAnnotator +from core.feed.types import FeedEntryType, Link, WorkEntry +from core.feed.util import strftime +from core.model import tuple_to_numericrange +from core.model.classification import Subject +from core.model.contributor import Contributor +from core.model.datasource import DataSource +from core.model.edition import Edition +from core.model.measurement import Measurement +from core.model.resource import Hyperlink, Resource +from core.model.work import Work +from core.util.datetime_helpers import utc_now +from tests.core.test_opds import TestAnnotatorsFixture, annotators_fixture # noqa +from tests.fixtures.database import ( # noqa + DatabaseTransactionFixture, + DBStatementCounter, +) + + +class TestAnnotators: + def test_all_subjects(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(genre="Fiction", with_open_access_download=True) + edition = work.presentation_edition + identifier = edition.primary_identifier + source1 = DataSource.lookup(session, DataSource.GUTENBERG) + source2 = DataSource.lookup(session, DataSource.OCLC) + + subjects = [ + (source1, Subject.FAST, "fast1", "name1", 1), + (source1, Subject.LCSH, "lcsh1", "name2", 1), + (source2, Subject.LCSH, "lcsh1", "name2", 1), + (source1, Subject.LCSH, "lcsh2", "name3", 3), + ( + source1, + Subject.DDC, + "300", + "Social sciences, sociology & anthropology", + 1, + ), + ] + + for source, subject_type, subject, name, weight in subjects: + identifier.classify(source, subject_type, subject, name, weight=weight) + + # Mock Work.all_identifier_ids (called by VerboseAnnotator.categories) + # so we can track the value that was passed in for `cutoff`. + def mock_all_identifier_ids(policy=None): + work.called_with_policy = policy + # Do the actual work so that categories() gets the + # correct information. + return work.original_all_identifier_ids(policy) + + work.original_all_identifier_ids = work.all_identifier_ids + work.all_identifier_ids = mock_all_identifier_ids + category_tags = VerboseAnnotator.categories(work) + + # When we are generating subjects as part of an OPDS feed, by + # default we set a cutoff of 100 equivalent identifiers. This + # gives us reasonable worst-case performance at the cost of + # not showing every single random subject under which an + # extremely popular book is filed. + assert 100 == work.called_with_policy.equivalent_identifier_cutoff + + ddc_uri = Subject.uri_lookup[Subject.DDC] + rating_value = "ratingValue" + assert [ + { + "term": "300", + rating_value: 1, + "label": "Social sciences, sociology & anthropology", + } + ] == category_tags[ddc_uri] + + fast_uri = Subject.uri_lookup[Subject.FAST] + assert [{"term": "fast1", "label": "name1", rating_value: 1}] == category_tags[ + fast_uri + ] + + lcsh_uri = Subject.uri_lookup[Subject.LCSH] + assert [ + {"term": "lcsh1", "label": "name2", rating_value: 2}, + {"term": "lcsh2", "label": "name3", rating_value: 3}, + ] == sorted(category_tags[lcsh_uri], key=lambda x: x[rating_value]) + + genre_uri = Subject.uri_lookup[Subject.SIMPLIFIED_GENRE] + assert [ + dict(label="Fiction", term=Subject.SIMPLIFIED_GENRE + "Fiction") + ] == category_tags[genre_uri] + + # Age range assertions + work = db.work(fiction=False, audience=Classifier.AUDIENCE_CHILDREN) + work.target_age = tuple_to_numericrange((8, 12)) + categories = Annotator.categories(work) + assert categories[Subject.SIMPLIFIED_FICTION_STATUS] == [ + dict( + term=f"{Subject.SIMPLIFIED_FICTION_STATUS}Nonfiction", + label="Nonfiction", + ) + ] + assert categories[Subject.uri_lookup[Subject.AGE_RANGE]] == [ + dict(term=work.target_age_string, label=work.target_age_string) + ] + + def test_content(self, db: DatabaseTransactionFixture): + work = db.work() + work.summary_text = "A Summary" + assert Annotator.content(work) == "A Summary" + + resrc = Resource() + db.session.add(resrc) + resrc.set_fetched_content("text", "Representation Summary", None) + + work.summary = resrc + work.summary_text = None + # The resource sets the summary + assert Annotator.content(work) == "Representation Summary" + assert work.summary_text == "Representation Summary" + + assert Annotator.content(None) == "" + + def test_appeals(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(with_open_access_download=True) + work.appeal_language = 0.1 + work.appeal_character = 0.2 + work.appeal_story = 0.3 + work.appeal_setting = 0.4 + work.calculate_opds_entries(verbose=True) + + category_tags = VerboseAnnotator.categories(work) + appeal_tags = category_tags[Work.APPEALS_URI] + expect = [ + (Work.APPEALS_URI + Work.LANGUAGE_APPEAL, Work.LANGUAGE_APPEAL, 0.1), + (Work.APPEALS_URI + Work.CHARACTER_APPEAL, Work.CHARACTER_APPEAL, 0.2), + (Work.APPEALS_URI + Work.STORY_APPEAL, Work.STORY_APPEAL, 0.3), + (Work.APPEALS_URI + Work.SETTING_APPEAL, Work.SETTING_APPEAL, 0.4), + ] + actual = [(x["term"], x["label"], x["ratingValue"]) for x in appeal_tags] + assert set(expect) == set(actual) + + def test_detailed_author(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + c, ignore = db.contributor("Familyname, Givenname") + c.display_name = "Givenname Familyname" + c.family_name = "Familyname" + c.wikipedia_name = "Givenname Familyname (Author)" + c.viaf = "100" + c.lc = "n100" + + author = VerboseAnnotator.detailed_author(c) + + assert "Givenname Familyname" == author.name + assert "Familyname, Givenname" == author.sort_name + assert "Givenname Familyname (Author)" == author.wikipedia_name + assert "http://viaf.org/viaf/100" == author.viaf + assert "http://id.loc.gov/authorities/names/n100" == author.lc + + work = db.work(authors=[], with_license_pool=True) + work.presentation_edition.add_contributor(c, Contributor.PRIMARY_AUTHOR_ROLE) + + [same_tag] = VerboseAnnotator.authors(work.presentation_edition)["authors"] + assert same_tag.dict() == author.dict() + + def test_duplicate_author_names_are_ignored( + self, annotators_fixture: TestAnnotatorsFixture + ): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + # Ignores duplicate author names + work = db.work(with_license_pool=True) + duplicate = db.contributor()[0] + duplicate.sort_name = work.author + + edition = work.presentation_edition + edition.add_contributor(duplicate, Contributor.AUTHOR_ROLE) + + assert 1 == len(Annotator.authors(edition)["authors"]) + + def test_all_annotators_mention_every_relevant_author( + self, annotators_fixture: TestAnnotatorsFixture + ): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(authors=[], with_license_pool=True) + edition = work.presentation_edition + + primary_author, ignore = db.contributor() + author, ignore = db.contributor() + illustrator, ignore = db.contributor() + barrel_washer, ignore = db.contributor() + + edition.add_contributor(primary_author, Contributor.PRIMARY_AUTHOR_ROLE) + edition.add_contributor(author, Contributor.AUTHOR_ROLE) + + # This contributor is relevant because we have a MARC Role Code + # for the role. + edition.add_contributor(illustrator, Contributor.ILLUSTRATOR_ROLE) + + # This contributor is not relevant because we have no MARC + # Role Code for the role. + edition.add_contributor(barrel_washer, "Barrel Washer") + + illustrator_code = Contributor.MARC_ROLE_CODES[Contributor.ILLUSTRATOR_ROLE] + + tags = Annotator.authors(edition) + # We made two tags and one + # tag, for the illustrator. + assert 2 == len(tags["authors"]) + assert 1 == len(tags["contributors"]) + assert [None, None, illustrator_code] == [ + x.role for x in (tags["authors"] + tags["contributors"]) + ] + + # Verbose annotator only creates author tags + tags = VerboseAnnotator.authors(edition) + assert 2 == len(tags["authors"]) + assert 0 == len(tags["contributors"]) + assert [None, None] == [x.role for x in (tags["authors"])] + + def test_ratings(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(with_license_pool=True, with_open_access_download=True) + work.quality = 1.0 / 3 + work.popularity = 0.25 + work.rating = 0.6 + work.calculate_opds_entries(verbose=True) + entry = OPDSAcquisitionFeed._create_entry( + work, + work.active_license_pool(), + work.presentation_edition, + work.presentation_edition.primary_identifier, + VerboseAnnotator(), + ) + assert entry.computed is not None + + ratings = [ + ( + getattr(rating, "ratingValue"), + getattr(rating, "additionalType"), + ) + for rating in entry.computed.ratings + ] + expected = [ + ("0.3333", Measurement.QUALITY), + ("0.2500", Measurement.POPULARITY), + ("0.6000", None), + ] + assert set(expected) == set(ratings) + + def test_subtitle(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(with_license_pool=True, with_open_access_download=True) + work.presentation_edition.subtitle = "Return of the Jedi" + work.calculate_opds_entries() + + feed = OPDSAcquisitionFeed( + db.fresh_str(), + db.fresh_url(), + [work], + CirculationManagerAnnotator(None), + )._feed + + computed = feed.entries[0].computed + assert computed is not None + assert computed.subtitle is not None + assert computed.subtitle.text == "Return of the Jedi" + + # If there's no subtitle, the subtitle tag isn't included. + work.presentation_edition.subtitle = None + work.calculate_opds_entries() + feed = OPDSAcquisitionFeed( + db.fresh_str(), + db.fresh_url(), + [work], + CirculationManagerAnnotator(None), + )._feed + + computed = feed.entries[0].computed + assert computed is not None + assert computed.subtitle == None + + def test_series(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(with_license_pool=True, with_open_access_download=True) + work.presentation_edition.series = "Harry Otter and the Lifetime of Despair" + work.presentation_edition.series_position = 4 + work.calculate_opds_entries() + + feed = OPDSAcquisitionFeed( + db.fresh_str(), + db.fresh_url(), + [work], + CirculationManagerAnnotator(None), + )._feed + computed = feed.entries[0].computed + assert computed is not None + + assert computed.series is not None + assert computed.series.name == work.presentation_edition.series # type: ignore[attr-defined] + assert computed.series.position == str( # type: ignore[attr-defined] + work.presentation_edition.series_position + ) + + # The series position can be 0, for a prequel for example. + work.presentation_edition.series_position = 0 + work.calculate_opds_entries() + + feed = OPDSAcquisitionFeed( + db.fresh_str(), + db.fresh_url(), + [work], + CirculationManagerAnnotator(None), + )._feed + computed = feed.entries[0].computed + assert computed is not None + assert computed.series is not None + assert computed.series.name == work.presentation_edition.series # type: ignore[attr-defined] + assert computed.series.position == str( # type: ignore[attr-defined] + work.presentation_edition.series_position + ) + + # If there's no series title, the series tag isn't included. + work.presentation_edition.series = None + work.calculate_opds_entries() + feed = OPDSAcquisitionFeed( + db.fresh_str(), + db.fresh_url(), + [work], + CirculationManagerAnnotator(None), + )._feed + computed = feed.entries[0].computed + assert computed is not None + assert computed.series == None + + # No series name + assert Annotator.series(None, "") == None + + def test_samples(self, annotators_fixture: TestAnnotatorsFixture): + data, db, session = ( + annotators_fixture, + annotators_fixture.db, + annotators_fixture.session, + ) + + work = db.work(with_license_pool=True) + edition = work.presentation_edition + + resource = Resource(url="sampleurl") + session.add(resource) + session.commit() + + sample_link = Hyperlink( + rel=Hyperlink.SAMPLE, + resource_id=resource.id, + identifier_id=edition.primary_identifier_id, + data_source_id=2, + ) + session.add(sample_link) + session.commit() + + with DBStatementCounter(db.database.connection) as counter: + links = Annotator.samples(edition) + count = counter.count + + assert len(links) == 1 + assert links[0].id == sample_link.id + assert links[0].resource.url == "sampleurl" + # accessing resource should not be another query + assert counter.count == count + + # No edition = No samples + assert Annotator.samples(None) == [] + + +class TestAnnotator: + def test_annotate_work_entry(self, db: DatabaseTransactionFixture): + work = db.work(with_license_pool=True) + pool = work.active_license_pool() + edition: Edition = work.presentation_edition + now = utc_now() + + edition.cover_full_url = "http://coverurl.jpg" + edition.cover_thumbnail_url = "http://thumburl.gif" + work.summary_text = "Summary" + edition.language = None + work.last_update_time = now + edition.publisher = "publisher" + edition.imprint = "imprint" + edition.issued = utc_now().date() + + # datetime for > today + pool.availability_time = (utc_now() + timedelta(days=1)).date() + + entry = WorkEntry( + work=work, + edition=edition, + identifier=edition.primary_identifier, + license_pool=pool, + ) + Annotator().annotate_work_entry(entry) + data = entry.computed + assert data is not None + + # Images + assert len(data.image_links) == 2 + assert data.image_links[0] == Link( + href=edition.cover_full_url, rel=Hyperlink.IMAGE, type="image/jpeg" + ) + assert data.image_links[1] == Link( + href=edition.cover_thumbnail_url, + rel=Hyperlink.THUMBNAIL_IMAGE, + type="image/gif", + ) + + # Other values + assert data.imprint == FeedEntryType(text="imprint") + assert data.summary and data.summary.text == "Summary" + assert data.summary and data.summary.get("type") == "html" + assert data.publisher == FeedEntryType(text="publisher") + assert data.issued == edition.issued + + # Missing values + assert data.language == None + assert data.updated == FeedEntryType(text=strftime(now)) diff --git a/tests/api/feed/test_library_annotator.py b/tests/api/feed/test_library_annotator.py new file mode 100644 index 0000000000..7c2cc3179d --- /dev/null +++ b/tests/api/feed/test_library_annotator.py @@ -0,0 +1,1795 @@ +import datetime +from collections import defaultdict +from typing import List +from unittest.mock import create_autospec + +import dateutil +import feedparser +import pytest +from freezegun import freeze_time +from lxml import etree + +from api.adobe_vendor_id import AuthdataUtility +from api.circulation import BaseCirculationAPI, CirculationAPI, FulfillmentInfo +from api.lanes import ContributorLane +from api.novelist import NoveListAPI +from core.analytics import Analytics +from core.classifier import ( # type: ignore[attr-defined] + Classifier, + Fantasy, + Urban_Fantasy, +) +from core.entrypoint import AudiobooksEntryPoint, EbooksEntryPoint, EverythingEntryPoint +from core.external_search import MockExternalSearchIndex +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.circulation import LibraryAnnotator +from core.feed.annotator.loan_and_hold import LibraryLoanAndHoldAnnotator +from core.feed.types import FeedData, WorkEntry +from core.feed.util import strftime +from core.lane import Facets, FacetsWithEntryPoint, Pagination +from core.lcp.credential import LCPCredentialFactory, LCPHashedPassphrase +from core.model import ( + CirculationEvent, + Contributor, + DataSource, + DeliveryMechanism, + ExternalIntegration, + Hyperlink, + PresentationCalculationPolicy, + Representation, + RightsStatus, + Work, +) +from core.opds import UnfulfillableWork +from core.opds_import import OPDSXMLParser +from core.util.datetime_helpers import utc_now +from core.util.flask_util import OPDSFeedResponse +from core.util.opds_writer import OPDSFeed +from tests.api.feed.fixtures import PatchedUrlFor, patch_url_for # noqa +from tests.fixtures.database import DatabaseTransactionFixture +from tests.fixtures.library import LibraryFixture +from tests.fixtures.vendor_id import VendorIDFixture + + +class LibraryAnnotatorFixture: + def __init__(self, db: DatabaseTransactionFixture): + self.db = db + self.work = db.work(with_open_access_download=True) + parent = db.lane(display_name="Fiction", languages=["eng"], fiction=True) + self.lane = db.lane(display_name="Fantasy", languages=["eng"]) + self.lane.add_genre(Fantasy.name) + self.lane.parent = parent + self.annotator = LibraryAnnotator( + None, + self.lane, + db.default_library(), + top_level_title="Test Top Level Title", + ) + + # Initialize library with Adobe Vendor ID details + db.default_library().library_registry_short_name = "FAKE" + db.default_library().library_registry_shared_secret = "s3cr3t5" + + # A ContributorLane to test code that handles it differently. + self.contributor, ignore = db.contributor("Someone") + self.contributor_lane = ContributorLane( + db.default_library(), self.contributor, languages=["eng"], audiences=None + ) + + +@pytest.fixture(scope="function") +def annotator_fixture( + db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor +) -> LibraryAnnotatorFixture: + return LibraryAnnotatorFixture(db) + + +class TestLibraryAnnotator: + def test_add_configuration_links( + self, + annotator_fixture: LibraryAnnotatorFixture, + library_fixture: LibraryFixture, + ): + mock_feed = FeedData() + + # Set up configuration settings for links. + library = annotator_fixture.db.default_library() + settings = library_fixture.settings(library) + settings.terms_of_service = "http://terms/" # type: ignore[assignment] + settings.privacy_policy = "http://privacy/" # type: ignore[assignment] + settings.copyright = "http://copyright/" # type: ignore[assignment] + settings.about = "http://about/" # type: ignore[assignment] + settings.license = "http://license/" # type: ignore[assignment] + settings.help_email = "help@me" # type: ignore[assignment] + settings.help_web = "http://help/" # type: ignore[assignment] + + # Set up settings for navigation links. + settings.web_header_links = ["http://example.com/1", "http://example.com/2"] + settings.web_header_labels = ["one", "two"] + + annotator_fixture.annotator.add_configuration_links(mock_feed) + + assert 9 == len(mock_feed.links) + + mock_feed_links = sorted(mock_feed.links, key=lambda x: x.rel or "") + expected_links = [ + (link.href, link.type) for link in mock_feed_links if link.rel != "related" + ] + + # They are the links we'd expect. + assert [ + ("http://about/", "text/html"), + ("http://copyright/", "text/html"), + ("mailto:help@me", None), + ("http://help/", "text/html"), + ("http://license/", "text/html"), + ("http://privacy/", "text/html"), + ("http://terms/", "text/html"), + ] == expected_links + + # There are two navigation links. + navigation_links = [x for x in mock_feed_links if x.rel == "related"] + assert {"navigation"} == {x.role for x in navigation_links} + assert {"http://example.com/1", "http://example.com/2"} == { + x.href for x in navigation_links + } + assert {"one", "two"} == {x.title for x in navigation_links} + + def test_top_level_title(self, annotator_fixture: LibraryAnnotatorFixture): + assert "Test Top Level Title" == annotator_fixture.annotator.top_level_title() + + def test_group_uri_with_flattened_lane( + self, annotator_fixture: LibraryAnnotatorFixture + ): + spanish_lane = annotator_fixture.db.lane( + display_name="Spanish", languages=["spa"] + ) + flat_spanish_lane = dict( + {"lane": spanish_lane, "label": "All Spanish", "link_to_list_feed": True} + ) + spanish_work = annotator_fixture.db.work( + title="Spanish Book", with_license_pool=True, language="spa" + ) + lp = spanish_work.license_pools[0] + annotator_fixture.annotator.lanes_by_work[spanish_work].append( + flat_spanish_lane + ) + + feed_url = annotator_fixture.annotator.feed_url(spanish_lane) + group_uri = annotator_fixture.annotator.group_uri( + spanish_work, lp, lp.identifier + ) + assert (feed_url, "All Spanish") == group_uri + + def test_lane_url(self, annotator_fixture: LibraryAnnotatorFixture): + fantasy_lane_with_sublanes = annotator_fixture.db.lane( + display_name="Fantasy with sublanes", languages=["eng"] + ) + fantasy_lane_with_sublanes.add_genre(Fantasy.name) + + urban_fantasy_lane = annotator_fixture.db.lane(display_name="Urban Fantasy") + urban_fantasy_lane.add_genre(Urban_Fantasy.name) + fantasy_lane_with_sublanes.sublanes.append(urban_fantasy_lane) + + fantasy_lane_without_sublanes = annotator_fixture.db.lane( + display_name="Fantasy without sublanes", languages=["eng"] + ) + fantasy_lane_without_sublanes.add_genre(Fantasy.name) + + default_lane_url = annotator_fixture.annotator.lane_url(None) + assert default_lane_url == annotator_fixture.annotator.default_lane_url() + + facets = FacetsWithEntryPoint(entrypoint=EbooksEntryPoint) + default_lane_url = annotator_fixture.annotator.lane_url(None, facets=facets) + assert default_lane_url == annotator_fixture.annotator.default_lane_url( + facets=facets + ) + + groups_url = annotator_fixture.annotator.lane_url(fantasy_lane_with_sublanes) + assert groups_url == annotator_fixture.annotator.groups_url( + fantasy_lane_with_sublanes + ) + + groups_url = annotator_fixture.annotator.lane_url( + fantasy_lane_with_sublanes, facets=facets + ) + assert groups_url == annotator_fixture.annotator.groups_url( + fantasy_lane_with_sublanes, facets=facets + ) + + feed_url = annotator_fixture.annotator.lane_url(fantasy_lane_without_sublanes) + assert feed_url == annotator_fixture.annotator.feed_url( + fantasy_lane_without_sublanes + ) + + feed_url = annotator_fixture.annotator.lane_url( + fantasy_lane_without_sublanes, facets=facets + ) + assert feed_url == annotator_fixture.annotator.feed_url( + fantasy_lane_without_sublanes, facets=facets + ) + + def test_fulfill_link_issues_only_open_access_links_when_library_does_not_identify_patrons( + self, annotator_fixture: LibraryAnnotatorFixture + ): + # This library doesn't identify patrons. + annotator_fixture.annotator.identifies_patrons = False + + # Because of this, normal fulfillment links are not generated. + [pool] = annotator_fixture.work.license_pools + [lpdm] = pool.delivery_mechanisms + assert None == annotator_fixture.annotator.fulfill_link(pool, None, lpdm) + + # However, fulfillment links _can_ be generated with the + # 'open-access' link relation. + link = annotator_fixture.annotator.fulfill_link( + pool, None, lpdm, OPDSFeed.OPEN_ACCESS_REL + ) + assert link is not None + assert OPDSFeed.OPEN_ACCESS_REL == link.rel + + # We freeze the test time here, because this test checks that the client token + # in the feed matches a generated client token. The client token contains an + # expiry date based on the current time, so this test can be flaky in a slow + # integration environment unless we make sure the clock does not change as this + # test is being performed. + @freeze_time("1867-07-01") + def test_fulfill_link_includes_device_registration_tags( + self, + annotator_fixture: LibraryAnnotatorFixture, + vendor_id_fixture: VendorIDFixture, + ): + """Verify that when Adobe Vendor ID delegation is included, the + fulfill link for an Adobe delivery mechanism includes instructions + on how to get a Vendor ID. + """ + vendor_id_fixture.initialize_adobe(annotator_fixture.db.default_library()) + [pool] = annotator_fixture.work.license_pools + identifier = pool.identifier + patron = annotator_fixture.db.patron() + old_credentials = list(patron.credentials) + + loan, ignore = pool.loan_to(patron, start=utc_now()) + adobe_delivery_mechanism, ignore = DeliveryMechanism.lookup( + annotator_fixture.db.session, "text/html", DeliveryMechanism.ADOBE_DRM + ) + other_delivery_mechanism, ignore = DeliveryMechanism.lookup( + annotator_fixture.db.session, "text/html", DeliveryMechanism.OVERDRIVE_DRM + ) + + # The fulfill link for non-Adobe DRM does not + # include the drm:licensor tag. + link = annotator_fixture.annotator.fulfill_link( + pool, loan, other_delivery_mechanism + ) + assert link is not None + for name, child in link: + assert name != "licensor" + + # No new Credential has been associated with the patron. + assert old_credentials == patron.credentials + + # The fulfill link for Adobe DRM includes information + # on how to get an Adobe ID in the drm:licensor tag. + link = annotator_fixture.annotator.fulfill_link( + pool, loan, adobe_delivery_mechanism + ) + licensor = getattr(link, "licensor", None) + assert None != licensor + + # An Adobe ID-specific identifier has been created for the patron. + [adobe_id_identifier] = [ + x for x in patron.credentials if x not in old_credentials + ] + assert ( + AuthdataUtility.ADOBE_ACCOUNT_ID_PATRON_IDENTIFIER + == adobe_id_identifier.type + ) + assert DataSource.INTERNAL_PROCESSING == adobe_id_identifier.data_source.name + assert None == adobe_id_identifier.expires + + # The drm:licensor tag is the one we get by calling + # adobe_id_tags() on that identifier. + assert adobe_id_identifier.credential is not None + expect = annotator_fixture.annotator.adobe_id_tags( + adobe_id_identifier.credential + ) + assert expect.get("licensor") == licensor + + def test_no_adobe_id_tags_when_vendor_id_not_configured( + self, annotator_fixture: LibraryAnnotatorFixture + ): + """When vendor ID delegation is not configured, adobe_id_tags() + returns an empty list. + """ + assert {} == annotator_fixture.annotator.adobe_id_tags("patron identifier") + + def test_adobe_id_tags_when_vendor_id_configured( + self, + annotator_fixture: LibraryAnnotatorFixture, + vendor_id_fixture: VendorIDFixture, + ): + """When vendor ID delegation is configured, adobe_id_tags() + returns a list containing a single tag. The tag contains + the information necessary to get an Adobe ID and a link to the local + DRM Device Management Protocol endpoint. + """ + library = annotator_fixture.db.default_library() + vendor_id_fixture.initialize_adobe(library) + patron_identifier = "patron identifier" + element = annotator_fixture.annotator.adobe_id_tags(patron_identifier) + + assert "licensor" in element + assert vendor_id_fixture.TEST_VENDOR_ID == getattr( + element["licensor"], "vendor", None + ) + + token = getattr(element["licensor"], "clientToken", None) + assert token is not None + # token.text is a token which we can decode, since we know + # the secret. + token_text = token.text + authdata = AuthdataUtility.from_config(library) + assert authdata is not None + decoded = authdata.decode_short_client_token(token_text) + expected_url = library.settings.website + assert (expected_url, patron_identifier) == decoded + + # If we call adobe_id_tags again we'll get a distinct tag + # object that renders to the same data. + same_tag = annotator_fixture.annotator.adobe_id_tags(patron_identifier) + assert same_tag is not element + assert same_tag["licensor"].dict() == element["licensor"].dict() + + # If the Adobe Vendor ID configuration is present but + # incomplete, adobe_id_tags does nothing. + + # Delete one setting from the existing integration to check + # this. + vendor_id_fixture.registration.short_name = None + assert {} == annotator_fixture.annotator.adobe_id_tags("new identifier") + + def test_lcp_acquisition_link_contains_hashed_passphrase( + self, annotator_fixture: LibraryAnnotatorFixture + ): + [pool] = annotator_fixture.work.license_pools + identifier = pool.identifier + patron = annotator_fixture.db.patron() + + hashed_password = LCPHashedPassphrase("hashed password") + + # Setup LCP credentials + lcp_credential_factory = LCPCredentialFactory() + lcp_credential_factory.set_hashed_passphrase( + annotator_fixture.db.session, patron, hashed_password + ) + + loan, ignore = pool.loan_to(patron, start=utc_now()) + lcp_delivery_mechanism, ignore = DeliveryMechanism.lookup( + annotator_fixture.db.session, "text/html", DeliveryMechanism.LCP_DRM + ) + other_delivery_mechanism, ignore = DeliveryMechanism.lookup( + annotator_fixture.db.session, "text/html", DeliveryMechanism.OVERDRIVE_DRM + ) + + # The fulfill link for non-LCP DRM does not include the hashed_passphrase tag. + link = annotator_fixture.annotator.fulfill_link( + pool, loan, other_delivery_mechanism + ) + assert not hasattr(link, "hashed_passphrase") + + # The fulfill link for lcp DRM includes hashed_passphrase + link = annotator_fixture.annotator.fulfill_link( + pool, loan, lcp_delivery_mechanism + ) + hashed_passphrase = getattr(link, "hashed_passphrase", None) + assert hashed_passphrase is not None + assert hashed_passphrase.text == hashed_password.hashed + + def test_default_lane_url(self, annotator_fixture: LibraryAnnotatorFixture): + default_lane_url = annotator_fixture.annotator.default_lane_url() + assert "groups" in default_lane_url + assert str(annotator_fixture.lane.id) not in default_lane_url + + facets = FacetsWithEntryPoint(entrypoint=EbooksEntryPoint) + default_lane_url = annotator_fixture.annotator.default_lane_url(facets=facets) + assert "entrypoint=Book" in default_lane_url + + def test_groups_url(self, annotator_fixture: LibraryAnnotatorFixture): + groups_url_no_lane = annotator_fixture.annotator.groups_url(None) + assert "groups" in groups_url_no_lane + assert str(annotator_fixture.lane.id) not in groups_url_no_lane + + groups_url_fantasy = annotator_fixture.annotator.groups_url( + annotator_fixture.lane + ) + assert "groups" in groups_url_fantasy + assert str(annotator_fixture.lane.id) in groups_url_fantasy + + facets = Facets.default( + annotator_fixture.db.default_library(), order="someorder" + ) + groups_url_facets = annotator_fixture.annotator.groups_url(None, facets=facets) + assert "order=someorder" in groups_url_facets + + def test_feed_url(self, annotator_fixture: LibraryAnnotatorFixture): + # A regular Lane. + feed_url_fantasy = annotator_fixture.annotator.feed_url( + annotator_fixture.lane, + Facets.default(annotator_fixture.db.default_library(), order="order"), + Pagination.default(), + ) + assert "feed" in feed_url_fantasy + assert "order=order" in feed_url_fantasy + assert str(annotator_fixture.lane.id) in feed_url_fantasy + + default_library = annotator_fixture.db.default_library() + assert default_library.name is not None + assert default_library.name in feed_url_fantasy + + # A QueryGeneratedLane. + annotator_fixture.annotator.lane = annotator_fixture.contributor_lane + feed_url_contributor = annotator_fixture.annotator.feed_url( + annotator_fixture.contributor_lane, + Facets.default(annotator_fixture.db.default_library()), + Pagination.default(), + ) + assert annotator_fixture.contributor_lane.ROUTE in feed_url_contributor + assert ( + annotator_fixture.contributor_lane.contributor_key in feed_url_contributor + ) + default_library = annotator_fixture.db.default_library() + assert default_library.name is not None + assert default_library.name in feed_url_contributor + + def test_search_url(self, annotator_fixture: LibraryAnnotatorFixture): + search_url = annotator_fixture.annotator.search_url( + annotator_fixture.lane, + "query", + Pagination.default(), + Facets.default(annotator_fixture.db.default_library(), order="Book"), + ) + assert "search" in search_url + assert "query" in search_url + assert "order=Book" in search_url + assert str(annotator_fixture.lane.id) in search_url + + def test_facet_url(self, annotator_fixture: LibraryAnnotatorFixture): + # A regular Lane. + facets = Facets.default( + annotator_fixture.db.default_library(), collection="main" + ) + facet_url = annotator_fixture.annotator.facet_url(facets) + assert "collection=main" in facet_url + assert str(annotator_fixture.lane.id) in facet_url + + # A QueryGeneratedLane. + annotator_fixture.annotator.lane = annotator_fixture.contributor_lane + + facet_url_contributor = annotator_fixture.annotator.facet_url(facets) + assert "collection=main" in facet_url_contributor + assert annotator_fixture.contributor_lane.ROUTE in facet_url_contributor + assert ( + annotator_fixture.contributor_lane.contributor_key in facet_url_contributor + ) + + def test_alternate_link_is_permalink( + self, annotator_fixture: LibraryAnnotatorFixture + ): + work = annotator_fixture.db.work(with_open_access_download=True) + works = annotator_fixture.db.session.query(Work) + annotator = LibraryAnnotator( + None, + annotator_fixture.lane, + annotator_fixture.db.default_library(), + ) + pool = annotator.active_licensepool_for(work) + + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + assert entry.computed is not None + assert pool is not None + assert entry.computed.identifier == pool.identifier.urn + + [(alternate, type)] = [ + (x.href, x.type) for x in entry.computed.other_links if x.rel == "alternate" + ] + permalink, permalink_type = annotator_fixture.annotator.permalink_for( + pool.identifier + ) + assert alternate == permalink + assert OPDSFeed.ENTRY_TYPE == type + assert permalink_type == type + + # Make sure we are using the 'permalink' controller -- we were using + # 'work' and that was wrong. + assert "/host/permalink" in permalink + + def test_annotate_work_entry(self, annotator_fixture: LibraryAnnotatorFixture): + lane = annotator_fixture.db.lane() + + # Create a Work. + work = annotator_fixture.db.work(with_license_pool=True) + [pool] = work.license_pools + identifier = pool.identifier + edition = pool.presentation_edition + + # Try building an entry for this Work with and without + # patron authentication turned on -- each setting is valid + # but will result in different links being available. + linksets = [] + for auth in (True, False): + annotator = LibraryAnnotator( + None, + lane, + annotator_fixture.db.default_library(), + library_identifies_patrons=auth, + ) + work_entry = WorkEntry( + work=work, + license_pool=pool, + edition=work.presentation_edition, + identifier=work.presentation_edition.primary_identifier, + ) + annotator.annotate_work_entry(work_entry) + + assert work_entry.computed is not None + linksets.append( + { + x.rel + for x in ( + work_entry.computed.other_links + + work_entry.computed.acquisition_links + ) + } + ) + + with_auth, no_auth = linksets + + # Some links are present no matter what. + for expect in ["alternate", "related"]: + assert expect in with_auth + assert expect in no_auth + + # A library with patron authentication offers some additional + # links -- one to borrow the book and one to annotate the + # book. + for expect in [ + "http://www.w3.org/ns/oa#annotationService", + "http://opds-spec.org/acquisition/borrow", + ]: + assert expect in with_auth + assert expect not in no_auth + + # We can also build an entry for a work with no license pool, + # but it will have no borrow link. + work = annotator_fixture.db.work(with_license_pool=False) + edition = work.presentation_edition + identifier = edition.primary_identifier + + annotator = LibraryAnnotator( + None, + lane, + annotator_fixture.db.default_library(), + library_identifies_patrons=True, + ) + work_entry = WorkEntry( + work=work, license_pool=None, edition=edition, identifier=identifier + ) + annotator.annotate_work_entry(work_entry) + assert work_entry.computed is not None + links = { + x.rel + for x in ( + work_entry.computed.other_links + work_entry.computed.acquisition_links + ) + } + + # These links are still present. + for expect in [ + "alternate", + "related", + "http://www.w3.org/ns/oa#annotationService", + ]: + assert expect in links + + # But the borrow link is gone. + assert "http://opds-spec.org/acquisition/borrow" not in links + + # There are no links to create analytics events for this title, + # because the library has no analytics configured. + open_book_rel = "http://librarysimplified.org/terms/rel/analytics/open-book" + assert open_book_rel not in links + + # If analytics are configured, a link is added to + # create an 'open_book' analytics event for this title. + Analytics.GLOBAL_ENABLED = True + work_entry = WorkEntry( + work=work, license_pool=None, edition=edition, identifier=identifier + ) + annotator.annotate_work_entry(work_entry) + assert work_entry.computed is not None + [analytics_link] = [ + x.href for x in work_entry.computed.other_links if x.rel == open_book_rel + ] + expect = annotator.url_for( + "track_analytics_event", + identifier_type=identifier.type, + identifier=identifier.identifier, + event_type=CirculationEvent.OPEN_BOOK, + library_short_name=annotator_fixture.db.default_library().short_name, + _external=True, + ) + assert expect == analytics_link + + # Test sample link with media types + link, _ = edition.primary_identifier.add_link( + Hyperlink.SAMPLE, + "http://example.org/sample", + edition.data_source, + media_type="application/epub+zip", + ) + work_entry = WorkEntry( + work=work, license_pool=None, edition=edition, identifier=identifier + ) + annotator.annotate_work_entry(work_entry) + assert work_entry.computed is not None + [feed_link] = [ + l + for l in work_entry.computed.other_links + if l.rel == Hyperlink.CLIENT_SAMPLE + ] + assert feed_link.href == link.resource.url + assert feed_link.type == link.resource.representation.media_type + + def test_annotate_feed(self, annotator_fixture: LibraryAnnotatorFixture): + lane = annotator_fixture.db.lane() + linksets = [] + for auth in (True, False): + annotator = LibraryAnnotator( + None, + lane, + annotator_fixture.db.default_library(), + library_identifies_patrons=auth, + ) + feed = OPDSAcquisitionFeed("test", "url", [], annotator) + annotator.annotate_feed(feed._feed) + linksets.append([x.rel for x in feed._feed.links]) + + with_auth, without_auth = linksets + + # There's always a a search link, and an auth + # document link. + for rel in ("search", "http://opds-spec.org/auth/document"): + assert rel in with_auth + assert rel in without_auth + + # But there's only a bookshelf link and an annotation link + # when patron authentication is enabled. + for rel in ( + "http://opds-spec.org/shelf", + "http://www.w3.org/ns/oa#annotationService", + ): + assert rel in with_auth + assert rel not in without_auth + + def get_parsed_feed( + self, annotator_fixture: LibraryAnnotatorFixture, works, lane=None, **kwargs + ): + if not lane: + lane = annotator_fixture.db.lane(display_name="Main Lane") + + feed = OPDSAcquisitionFeed( + "url", + "test", + works, + LibraryAnnotator( + None, + lane, + annotator_fixture.db.default_library(), + **kwargs, + ), + facets=FacetsWithEntryPoint(), + pagination=Pagination.default(), + ) + feed.generate_feed() + return feed._feed + + def assert_link_on_entry( + self, entry, link_type=None, rels=None, partials_by_rel=None + ): + """Asserts that a link with a certain 'rel' value exists on a + given feed or entry, as well as its link 'type' value and parts + of its 'href' value. + """ + + def get_link_by_rel(rel): + if isinstance(entry, WorkEntry): + links = entry.computed.other_links + entry.computed.acquisition_links + elif isinstance(entry, List): + links = [e.link for e in entry] + else: + links = [entry.link] + try: + [link] = [x for x in links if x.rel == rel] + except ValueError as e: + raise AssertionError + if link_type: + assert link_type == link.type + return link + + if rels: + [get_link_by_rel(rel) for rel in rels] + + partials_by_rel = partials_by_rel or dict() + for rel, uri_partials in list(partials_by_rel.items()): + link = get_link_by_rel(rel) + if not isinstance(uri_partials, list): + uri_partials = [uri_partials] + for part in uri_partials: + assert part in link.href + + def test_work_entry_includes_open_access_or_borrow_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + open_access_work = annotator_fixture.db.work(with_open_access_download=True) + licensed_work = annotator_fixture.db.work(with_license_pool=True) + licensed_work.license_pools[0].open_access = False + + feed = self.get_parsed_feed( + annotator_fixture, [open_access_work, licensed_work] + ) + [open_access_entry, licensed_entry] = feed.entries + + self.assert_link_on_entry(open_access_entry, rels=[OPDSFeed.BORROW_REL]) + self.assert_link_on_entry(licensed_entry, rels=[OPDSFeed.BORROW_REL]) + + def test_language_and_audience_key_from_work( + self, annotator_fixture: LibraryAnnotatorFixture + ): + work = annotator_fixture.db.work( + language="eng", audience=Classifier.AUDIENCE_CHILDREN + ) + result = annotator_fixture.annotator.language_and_audience_key_from_work(work) + assert ("eng", "Children") == result + + work = annotator_fixture.db.work( + language="fre", audience=Classifier.AUDIENCE_YOUNG_ADULT + ) + result = annotator_fixture.annotator.language_and_audience_key_from_work(work) + assert ("fre", "All+Ages,Children,Young+Adult") == result + + work = annotator_fixture.db.work( + language="spa", audience=Classifier.AUDIENCE_ADULT + ) + result = annotator_fixture.annotator.language_and_audience_key_from_work(work) + assert ("spa", "Adult,Adults+Only,All+Ages,Children,Young+Adult") == result + + work = annotator_fixture.db.work(audience=Classifier.AUDIENCE_ADULTS_ONLY) + result = annotator_fixture.annotator.language_and_audience_key_from_work(work) + assert ("eng", "Adult,Adults+Only,All+Ages,Children,Young+Adult") == result + + work = annotator_fixture.db.work(audience=Classifier.AUDIENCE_RESEARCH) + result = annotator_fixture.annotator.language_and_audience_key_from_work(work) + assert ( + "eng", + "Adult,Adults+Only,All+Ages,Children,Research,Young+Adult", + ) == result + + work = annotator_fixture.db.work(audience=Classifier.AUDIENCE_ALL_AGES) + result = annotator_fixture.annotator.language_and_audience_key_from_work(work) + assert ("eng", "All+Ages,Children") == result + + def test_work_entry_includes_contributor_links( + self, annotator_fixture: LibraryAnnotatorFixture + ): + """ContributorLane links are added to works with contributors""" + work = annotator_fixture.db.work(with_open_access_download=True) + contributor1 = work.presentation_edition.author_contributors[0] + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + + expected_rel_and_partial = dict(contributor="/contributor") + self.assert_link_on_entry( + entry.computed.authors, + link_type=OPDSFeed.ACQUISITION_FEED_TYPE, + partials_by_rel=expected_rel_and_partial, + ) + + # When there are two authors, they each get a contributor link. + work.presentation_edition.add_contributor("Oprah", Contributor.AUTHOR_ROLE) + work.calculate_presentation( + PresentationCalculationPolicy(regenerate_opds_entries=True), + MockExternalSearchIndex(), + ) + [entry] = self.get_parsed_feed(annotator_fixture, [work]).entries + contributor_links = [ + l.link for l in entry.computed.authors if hasattr(l, "link") + ] + assert 2 == len(contributor_links) + contributor_links.sort(key=lambda l: l.href) + for l in contributor_links: + assert l.type == OPDSFeed.ACQUISITION_FEED_TYPE + assert "/contributor" in l.href + assert contributor1.sort_name in contributor_links[0].href + assert "Oprah" in contributor_links[1].href + + # When there's no author, there's no contributor link. + annotator_fixture.db.session.delete(work.presentation_edition.contributions[0]) + annotator_fixture.db.session.delete(work.presentation_edition.contributions[1]) + annotator_fixture.db.session.commit() + work.calculate_presentation( + PresentationCalculationPolicy(regenerate_opds_entries=True), + MockExternalSearchIndex(), + ) + [entry] = self.get_parsed_feed(annotator_fixture, [work]).entries + assert [] == [l.link for l in entry.computed.authors if l.link] + + def test_work_entry_includes_series_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + """A series lane link is added to the work entry when its in a series""" + work = annotator_fixture.db.work( + with_open_access_download=True, series="Serious Cereals Series" + ) + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + expected_rel_and_partial = dict(series="/series") + self.assert_link_on_entry( + entry.computed.series, + link_type=OPDSFeed.ACQUISITION_FEED_TYPE, + partials_by_rel=expected_rel_and_partial, + ) + + # When there's no series, there's no series link. + work = annotator_fixture.db.work(with_open_access_download=True) + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + assert None == entry.computed.series + + def test_work_entry_includes_recommendations_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + work = annotator_fixture.db.work(with_open_access_download=True) + + # If NoveList Select isn't configured, there's no recommendations link. + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + assert [] == [ + l for l in entry.computed.other_links if l.rel == "recommendations" + ] + + # There's a recommendation link when configuration is found, though! + NoveListAPI.IS_CONFIGURED = None + annotator_fixture.db.external_integration( + ExternalIntegration.NOVELIST, + goal=ExternalIntegration.METADATA_GOAL, + username="library", + password="sure", + libraries=[annotator_fixture.db.default_library()], + ) + + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + expected_rel_and_partial = dict(recommendations="/recommendations") + self.assert_link_on_entry( + entry, + link_type=OPDSFeed.ACQUISITION_FEED_TYPE, + partials_by_rel=expected_rel_and_partial, + ) + + def test_work_entry_includes_annotations_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + work = annotator_fixture.db.work(with_open_access_download=True) + identifier_str = work.license_pools[0].identifier.identifier + uri_parts = ["/annotations", identifier_str] + annotation_rel = "http://www.w3.org/ns/oa#annotationService" + rel_with_partials = {annotation_rel: uri_parts} + + feed = self.get_parsed_feed(annotator_fixture, [work]) + [entry] = feed.entries + self.assert_link_on_entry(entry, partials_by_rel=rel_with_partials) + + # If the library does not authenticate patrons, no link to the + # annotation service is provided. + feed = self.get_parsed_feed( + annotator_fixture, [work], library_identifies_patrons=False + ) + [entry] = feed.entries + assert annotation_rel not in [x.rel for x in entry.computed.other_links] + + def test_active_loan_feed( + self, + annotator_fixture: LibraryAnnotatorFixture, + vendor_id_fixture: VendorIDFixture, + ): + vendor_id_fixture.initialize_adobe(annotator_fixture.db.default_library()) + patron = annotator_fixture.db.patron() + patron.last_loan_activity_sync = utc_now() + annotator = LibraryLoanAndHoldAnnotator( + None, + annotator_fixture.lane, + annotator_fixture.db.default_library(), + patron=patron, + ) + + response = OPDSAcquisitionFeed.active_loans_for( + None, patron, annotator + ).as_response() + + # The feed is private and should not be cached. + assert isinstance(response, OPDSFeedResponse) + + # No entries in the feed... + raw = str(response) + feed = feedparser.parse(raw) + assert 0 == len(feed["entries"]) + + # ... but we have a link to the User Profile Management + # Protocol endpoint... + links = feed["feed"]["links"] + [upmp_link] = [ + x + for x in links + if x["rel"] == "http://librarysimplified.org/terms/rel/user-profile" + ] + annotator = LibraryLoanAndHoldAnnotator( + None, None, library=patron.library, patron=patron + ) + expect_url = annotator.url_for( + "patron_profile", + library_short_name=patron.library.short_name, + _external=True, + ) + assert expect_url == upmp_link["href"] + + # ... and we have DRM licensing information. + tree = etree.fromstring(response.get_data(as_text=True)) + parser = OPDSXMLParser() + licensor = parser._xpath1(tree, "//atom:feed/drm:licensor") + + adobe_patron_identifier = AuthdataUtility._adobe_patron_identifier(patron) + + # The DRM licensing information includes the Adobe vendor ID + # and the patron's patron identifier for Adobe purposes. + assert ( + vendor_id_fixture.TEST_VENDOR_ID + == licensor.attrib["{http://librarysimplified.org/terms/drm}vendor"] + ) + [client_token] = licensor + assert vendor_id_fixture.registration.short_name is not None + expected = vendor_id_fixture.registration.short_name.upper() + assert client_token.text.startswith(expected) + assert adobe_patron_identifier in client_token.text + + # Unlike other places this tag shows up, we use the + # 'scheme' attribute to explicitly state that this + # tag is talking about an ACS licensing + # scheme. Since we're in a and not a to a + # specific book, that context would otherwise be lost. + assert ( + "http://librarysimplified.org/terms/drm/scheme/ACS" + == licensor.attrib["{http://librarysimplified.org/terms/drm}scheme"] + ) + + # Since we're taking a round trip to and from OPDS, which only + # represents times with second precision, generate the current + # time with second precision to make later comparisons + # possible. + now = utc_now().replace(microsecond=0) + tomorrow = now + datetime.timedelta(days=1) + + # A loan of an open-access book is open-ended. + work1 = annotator_fixture.db.work( + language="eng", with_open_access_download=True + ) + loan1 = work1.license_pools[0].loan_to(patron, start=now) + + # A loan of some other kind of book has an end point. + work2 = annotator_fixture.db.work(language="eng", with_license_pool=True) + loan2 = work2.license_pools[0].loan_to(patron, start=now, end=tomorrow) + unused = annotator_fixture.db.work( + language="eng", with_open_access_download=True + ) + + # Get the feed. + feed_obj = OPDSAcquisitionFeed.active_loans_for( + None, + patron, + LibraryLoanAndHoldAnnotator( + None, + annotator_fixture.lane, + annotator_fixture.db.default_library(), + patron=patron, + ), + ).as_response() + raw = str(feed_obj) + feed = feedparser.parse(raw) + + # The only entries in the feed is the work currently out on loan + # to this patron. + assert 2 == len(feed["entries"]) + e1, e2 = sorted(feed["entries"], key=lambda x: x["title"]) + assert work1.title == e1["title"] + assert work2.title == e2["title"] + + # Make sure that the start and end dates from the loan are present + # in an child of the acquisition link. + tree = etree.fromstring(raw) + parser = OPDSXMLParser() + acquisitions = parser._xpath( + tree, "//atom:entry/atom:link[@rel='http://opds-spec.org/acquisition']" + ) + assert 2 == len(acquisitions) + + availabilities = [parser._xpath1(x, "opds:availability") for x in acquisitions] + + # One of these availability tags has 'since' but not 'until'. + # The other one has both. + [no_until] = [x for x in availabilities if "until" not in x.attrib] + assert now == dateutil.parser.parse(no_until.attrib["since"]) + + [has_until] = [x for x in availabilities if "until" in x.attrib] + assert now == dateutil.parser.parse(has_until.attrib["since"]) + assert tomorrow == dateutil.parser.parse(has_until.attrib["until"]) + + def test_loan_feed_includes_patron( + self, annotator_fixture: LibraryAnnotatorFixture + ): + patron = annotator_fixture.db.patron() + + patron.username = "bellhooks" + patron.authorization_identifier = "987654321" + feed_obj = OPDSAcquisitionFeed.active_loans_for( + None, + patron, + LibraryLoanAndHoldAnnotator( + None, None, annotator_fixture.db.default_library(), patron + ), + ).as_response() + raw = str(feed_obj) + feed_details = feedparser.parse(raw)["feed"] + + assert "simplified:authorizationIdentifier" in raw + assert "simplified:username" in raw + assert ( + patron.username == feed_details["simplified_patron"]["simplified:username"] + ) + assert ( + "987654321" + == feed_details["simplified_patron"]["simplified:authorizationidentifier"] + ) + + def test_loans_feed_includes_annotations_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + patron = annotator_fixture.db.patron() + feed_obj = OPDSAcquisitionFeed.active_loans_for(None, patron).as_response() + raw = str(feed_obj) + feed = feedparser.parse(raw)["feed"] + links = feed["links"] + + [annotations_link] = [ + x + for x in links + if x["rel"].lower() == "http://www.w3.org/ns/oa#annotationService".lower() + ] + assert "/annotations" in annotations_link["href"] + + def test_active_loan_feed_ignores_inconsistent_local_data( + self, annotator_fixture: LibraryAnnotatorFixture + ): + patron = annotator_fixture.db.patron() + + work1 = annotator_fixture.db.work(language="eng", with_license_pool=True) + loan, ignore = work1.license_pools[0].loan_to(patron) + work2 = annotator_fixture.db.work(language="eng", with_license_pool=True) + hold, ignore = work2.license_pools[0].on_hold_to(patron) + + # Uh-oh, our local loan data is bad. + loan.license_pool.identifier = None + + # Our local hold data is also bad. + hold.license_pool = None + + # We can still get a feed... + feed_obj = OPDSAcquisitionFeed.active_loans_for(None, patron).as_response() + + # ...but it's empty. + assert "" not in str(feed_obj) + + def test_acquisition_feed_includes_license_information( + self, annotator_fixture: LibraryAnnotatorFixture + ): + work = annotator_fixture.db.work(with_open_access_download=True) + pool = work.license_pools[0] + + # These numbers are impossible, but it doesn't matter for + # purposes of this test. + pool.open_access = False + pool.licenses_owned = 100 + pool.licenses_available = 50 + pool.patrons_in_hold_queue = 25 + + work_entry = WorkEntry( + work=work, + license_pool=pool, + edition=work.presentation_edition, + identifier=work.presentation_edition.primary_identifier, + ) + annotator_fixture.annotator.annotate_work_entry(work_entry) + assert work_entry.computed is not None + [link] = work_entry.computed.acquisition_links + assert link.holds_total == "25" + + assert link.copies_available == "50" + assert link.copies_total == "100" + + def test_loans_feed_includes_fulfill_links( + self, + annotator_fixture: LibraryAnnotatorFixture, + library_fixture: LibraryFixture, + ): + patron = annotator_fixture.db.patron() + + work = annotator_fixture.db.work( + with_license_pool=True, with_open_access_download=False + ) + pool = work.license_pools[0] + pool.open_access = False + mech1 = pool.delivery_mechanisms[0] + mech2 = pool.set_delivery_mechanism( + Representation.PDF_MEDIA_TYPE, + DeliveryMechanism.ADOBE_DRM, + RightsStatus.IN_COPYRIGHT, + None, + ) + streaming_mech = pool.set_delivery_mechanism( + DeliveryMechanism.STREAMING_TEXT_CONTENT_TYPE, + DeliveryMechanism.OVERDRIVE_DRM, + RightsStatus.IN_COPYRIGHT, + None, + ) + + now = utc_now() + loan, ignore = pool.loan_to(patron, start=now) + + feed_obj = OPDSAcquisitionFeed.active_loans_for( + None, + patron, + ).as_response() + raw = str(feed_obj) + + entries = feedparser.parse(raw)["entries"] + assert 1 == len(entries) + + links = entries[0]["links"] + + # Before we fulfill the loan, there are fulfill links for all three mechanisms. + fulfill_links = [ + link for link in links if link["rel"] == "http://opds-spec.org/acquisition" + ] + assert 3 == len(fulfill_links) + + assert { + mech1.delivery_mechanism.drm_scheme_media_type, + mech2.delivery_mechanism.drm_scheme_media_type, + OPDSFeed.ENTRY_TYPE, + } == {link["type"] for link in fulfill_links} + + # If one of the content types is hidden, the corresponding + # delivery mechanism does not have a link. + library = annotator_fixture.db.default_library() + settings = library_fixture.settings(library) + settings.hidden_content_types = [mech1.delivery_mechanism.content_type] + OPDSAcquisitionFeed.active_loans_for(None, patron).as_response() + assert { + mech2.delivery_mechanism.drm_scheme_media_type, + OPDSFeed.ENTRY_TYPE, + } == {link["type"] for link in fulfill_links} + settings.hidden_content_types = [] + + # When the loan is fulfilled, there are only fulfill links for that mechanism + # and the streaming mechanism. + loan.fulfillment = mech1 + + feed_obj = OPDSAcquisitionFeed.active_loans_for(None, patron).as_response() + raw = str(feed_obj) + + entries = feedparser.parse(raw)["entries"] + assert 1 == len(entries) + + links = entries[0]["links"] + + fulfill_links = [ + link for link in links if link["rel"] == "http://opds-spec.org/acquisition" + ] + assert 2 == len(fulfill_links) + + assert { + mech1.delivery_mechanism.drm_scheme_media_type, + OPDSFeed.ENTRY_TYPE, + } == {link["type"] for link in fulfill_links} + + def test_incomplete_catalog_entry_contains_an_alternate_link_to_the_complete_entry( + self, annotator_fixture: LibraryAnnotatorFixture + ): + circulation = create_autospec(spec=CirculationAPI) + circulation.library = annotator_fixture.db.default_library() + work = annotator_fixture.db.work( + with_license_pool=True, with_open_access_download=False + ) + pool = work.license_pools[0] + + annotator = LibraryLoanAndHoldAnnotator( + circulation, annotator_fixture.lane, circulation.library + ) + + feed_obj = OPDSAcquisitionFeed.single_entry_loans_feed( + circulation, pool, annotator + ) + raw = str(feed_obj) + entries = feedparser.parse(raw)["entries"] + assert 1 == len(entries) + + links = entries[0]["links"] + + # We want to make sure that an incomplete catalog entry contains an alternate link to the complete entry. + alternate_links = [ + link + for link in links + if link["type"] == OPDSFeed.ENTRY_TYPE and link["rel"] == "alternate" + ] + assert 1 == len(alternate_links) + + def test_complete_catalog_entry_with_fulfillment_link_contains_self_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + patron = annotator_fixture.db.patron() + circulation = create_autospec(spec=CirculationAPI) + circulation.library = annotator_fixture.db.default_library() + work = annotator_fixture.db.work( + with_license_pool=True, with_open_access_download=False + ) + pool = work.license_pools[0] + loan, _ = pool.loan_to(patron) + + annotator = LibraryLoanAndHoldAnnotator(circulation, None, circulation.library) + feed_obj = OPDSAcquisitionFeed.single_entry_loans_feed( + circulation, loan, annotator + ) + raw = str(feed_obj) + + entries = feedparser.parse(raw)["entries"] + assert 1 == len(entries) + + links = entries[0]["links"] + + # We want to make sure that a complete catalog entry contains an alternate link + # because it's required by some clients (for example, an Android version of SimplyE). + alternate_links = [ + link + for link in links + if link["type"] == OPDSFeed.ENTRY_TYPE and link["rel"] == "alternate" + ] + assert 1 == len(alternate_links) + + # We want to make sure that the complete catalog entry contains a self link. + self_links = [ + link + for link in links + if link["type"] == OPDSFeed.ENTRY_TYPE and link["rel"] == "self" + ] + assert 1 == len(self_links) + + # We want to make sure that alternate and self links are the same. + assert alternate_links[0]["href"] == self_links[0]["href"] + + def test_complete_catalog_entry_with_fulfillment_info_contains_self_link( + self, annotator_fixture: LibraryAnnotatorFixture + ): + patron = annotator_fixture.db.patron() + circulation = create_autospec(spec=CirculationAPI) + circulation.library = annotator_fixture.db.default_library() + work = annotator_fixture.db.work( + with_license_pool=True, with_open_access_download=False + ) + pool = work.license_pools[0] + loan, _ = pool.loan_to(patron) + fulfillment = FulfillmentInfo( + pool.collection, + pool.data_source.name, + pool.identifier.type, + pool.identifier.identifier, + "http://link", + Representation.EPUB_MEDIA_TYPE, + None, + None, + ) + + annotator = LibraryLoanAndHoldAnnotator(circulation, None, circulation.library) + feed_obj = OPDSAcquisitionFeed.single_entry_loans_feed( + circulation, + loan, + annotator, + fulfillment=fulfillment, + ) + raw = str(feed_obj) + + entries = feedparser.parse(raw)["entries"] + assert 1 == len(entries) + + links = entries[0]["links"] + + # We want to make sure that a complete catalog entry contains an alternate link + # because it's required by some clients (for example, an Android version of SimplyE). + alternate_links = [ + link + for link in links + if link["type"] == OPDSFeed.ENTRY_TYPE and link["rel"] == "alternate" + ] + assert 1 == len(alternate_links) + + # We want to make sure that the complete catalog entry contains a self link. + self_links = [ + link + for link in links + if link["type"] == OPDSFeed.ENTRY_TYPE and link["rel"] == "self" + ] + assert 1 == len(self_links) + + # We want to make sure that alternate and self links are the same. + assert alternate_links[0]["href"] == self_links[0]["href"] + + def test_fulfill_feed(self, annotator_fixture: LibraryAnnotatorFixture): + patron = annotator_fixture.db.patron() + + work = annotator_fixture.db.work( + with_license_pool=True, with_open_access_download=False + ) + pool = work.license_pools[0] + pool.open_access = False + streaming_mech = pool.set_delivery_mechanism( + DeliveryMechanism.STREAMING_TEXT_CONTENT_TYPE, + DeliveryMechanism.OVERDRIVE_DRM, + RightsStatus.IN_COPYRIGHT, + None, + ) + + now = utc_now() + loan, ignore = pool.loan_to(patron, start=now) + fulfillment = FulfillmentInfo( + pool.collection, + pool.data_source.name, + pool.identifier.type, + pool.identifier.identifier, + "http://streaming_link", + Representation.TEXT_HTML_MEDIA_TYPE + DeliveryMechanism.STREAMING_PROFILE, + None, + None, + ) + + annotator = LibraryLoanAndHoldAnnotator(None, None, patron.library) + feed_obj = OPDSAcquisitionFeed.single_entry_loans_feed( + None, loan, annotator, fulfillment=fulfillment + ) + + entries = feedparser.parse(str(feed_obj))["entries"] + assert 1 == len(entries) + + links = entries[0]["links"] + + # The feed for a single fulfillment only includes one fulfill link. + fulfill_links = [ + link for link in links if link["rel"] == "http://opds-spec.org/acquisition" + ] + assert 1 == len(fulfill_links) + + assert ( + Representation.TEXT_HTML_MEDIA_TYPE + DeliveryMechanism.STREAMING_PROFILE + == fulfill_links[0]["type"] + ) + assert "http://streaming_link" == fulfill_links[0]["href"] + + def test_drm_device_registration_feed_tags( + self, + annotator_fixture: LibraryAnnotatorFixture, + vendor_id_fixture: VendorIDFixture, + ): + """Check that drm_device_registration_feed_tags returns + a generic drm:licensor tag, except with the drm:scheme attribute + set. + """ + vendor_id_fixture.initialize_adobe(annotator_fixture.db.default_library()) + annotator = LibraryLoanAndHoldAnnotator( + None, + None, + annotator_fixture.db.default_library(), + ) + patron = annotator_fixture.db.patron() + feed_tag = annotator.drm_device_registration_feed_tags(patron) + generic_tag = annotator.adobe_id_tags(patron) + + # The feed-level tag has the drm:scheme attribute set. + assert ( + "http://librarysimplified.org/terms/drm/scheme/ACS" + == feed_tag["licensor"].scheme + ) + + # If we remove that attribute, the feed-level tag is the same as the + # generic tag. + assert feed_tag["licensor"].dict() != generic_tag["licensor"].dict() + delattr(feed_tag["licensor"], "scheme") + assert feed_tag["licensor"].dict() == generic_tag["licensor"].dict() + + def test_borrow_link_raises_unfulfillable_work( + self, annotator_fixture: LibraryAnnotatorFixture + ): + edition, pool = annotator_fixture.db.edition(with_license_pool=True) + kindle_mechanism = pool.set_delivery_mechanism( + DeliveryMechanism.KINDLE_CONTENT_TYPE, + DeliveryMechanism.KINDLE_DRM, + RightsStatus.IN_COPYRIGHT, + None, + ) + epub_mechanism = pool.set_delivery_mechanism( + Representation.EPUB_MEDIA_TYPE, + DeliveryMechanism.ADOBE_DRM, + RightsStatus.IN_COPYRIGHT, + None, + ) + data_source_name = pool.data_source.name + identifier = pool.identifier + + annotator = LibraryLoanAndHoldAnnotator( + None, None, annotator_fixture.db.default_library() + ) + + # If there's no way to fulfill the book, borrow_link raises + # UnfulfillableWork. + pytest.raises(UnfulfillableWork, annotator.borrow_link, pool, None, []) + + pytest.raises( + UnfulfillableWork, annotator.borrow_link, pool, None, [kindle_mechanism] + ) + + # If there's a fulfillable mechanism, everything's fine. + link = annotator.borrow_link(pool, None, [epub_mechanism]) + assert link != None + + link = annotator.borrow_link(pool, None, [epub_mechanism, kindle_mechanism]) + assert link != None + + def test_feed_includes_lane_links(self, annotator_fixture: LibraryAnnotatorFixture): + def annotated_links(lane, annotator): + # Create an AcquisitionFeed is using the given Annotator. + # extract its links and return a dictionary that maps link + # relations to URLs. + feed = OPDSAcquisitionFeed("test", "url", [], annotator) + annotator.annotate_feed(feed._feed) + links = feed._feed.links + + d = defaultdict(list) + for link in links: + d[link.rel.lower()].append(link.href) + return d + + # When an EntryPoint is explicitly selected, it shows up in the + # link to the search controller. + facets = FacetsWithEntryPoint(entrypoint=AudiobooksEntryPoint) + lane = annotator_fixture.db.lane() + annotator = LibraryAnnotator( + None, + lane, + annotator_fixture.db.default_library(), + facets=facets, + ) + [url] = annotated_links(lane, annotator)["search"] + assert "/lane_search" in url + assert "entrypoint=%s" % AudiobooksEntryPoint.INTERNAL_NAME in url + assert str(lane.id) in url + + # When the selected EntryPoint is a default, it's not used -- + # instead, we search everything. + assert annotator.facets is not None + annotator.facets.entrypoint_is_default = True + links = annotated_links(lane, annotator) + [url] = links["search"] + assert "entrypoint=%s" % EverythingEntryPoint.INTERNAL_NAME in url + + # This lane isn't based on a custom list, so there's no crawlable link. + assert [] == links["http://opds-spec.org/crawlable"] + + # It's also not crawlable if it's based on multiple lists. + list1, ignore = annotator_fixture.db.customlist() + list2, ignore = annotator_fixture.db.customlist() + lane.customlists = [list1, list2] + links = annotated_links(lane, annotator) + assert [] == links["http://opds-spec.org/crawlable"] + + # A lane based on a single list gets a crawlable link. + lane.customlists = [list1] + links = annotated_links(lane, annotator) + [crawlable] = links["http://opds-spec.org/crawlable"] + assert "/crawlable_list_feed" in crawlable + assert str(list1.name) in crawlable + + def test_acquisition_links( + self, + annotator_fixture: LibraryAnnotatorFixture, + library_fixture: LibraryFixture, + ): + annotator = LibraryLoanAndHoldAnnotator( + None, None, annotator_fixture.db.default_library() + ) + + patron = annotator_fixture.db.patron() + + now = utc_now() + tomorrow = now + datetime.timedelta(days=1) + + # Loan of an open-access book. + work1 = annotator_fixture.db.work(with_open_access_download=True) + loan1, ignore = work1.license_pools[0].loan_to(patron, start=now) + + # Loan of a licensed book. + work2 = annotator_fixture.db.work(with_license_pool=True) + loan2, ignore = work2.license_pools[0].loan_to(patron, start=now, end=tomorrow) + + # Hold on a licensed book. + work3 = annotator_fixture.db.work(with_license_pool=True) + hold, ignore = work3.license_pools[0].on_hold_to( + patron, start=now, end=tomorrow + ) + + # Book with no loans or holds yet. + work4 = annotator_fixture.db.work(with_license_pool=True) + + # Loan of a licensed book without a loan end. + work5 = annotator_fixture.db.work(with_license_pool=True) + loan5, ignore = work5.license_pools[0].loan_to(patron, start=now) + + # Ensure the state variable + assert annotator.identifies_patrons == True + + loan1_links = annotator.acquisition_links( + loan1.license_pool, + loan1, + None, + None, + loan1.license_pool.identifier, + ) + # Fulfill, and revoke. + [revoke, fulfill] = sorted(loan1_links, key=lambda x: x.rel or "") + assert revoke.href and "revoke_loan_or_hold" in revoke.href + assert ( + revoke.rel and "http://librarysimplified.org/terms/rel/revoke" == revoke.rel + ) + assert fulfill.href and "fulfill" in fulfill.href + assert fulfill.rel and "http://opds-spec.org/acquisition" == fulfill.rel + + # Allow direct open-access downloads + # This will also filter out loan revoke links + annotator.identifies_patrons = False + loan1_links = annotator.acquisition_links( + loan1.license_pool, loan1, None, None, loan1.license_pool.identifier + ) + assert len(loan1_links) == 1 + assert {"http://opds-spec.org/acquisition/open-access"} == { + link.rel for link in loan1_links + } + + # Work 2 has no open access links + loan2_links = annotator.acquisition_links( + loan2.license_pool, loan2, None, None, loan2.license_pool.identifier + ) + assert len(loan2_links) == 0 + + # Revert the annotator state + annotator.identifies_patrons = True + + assert strftime(loan1.start) == fulfill.availability_since + assert loan1.end == fulfill.availability_until == None + + loan2_links = annotator.acquisition_links( + loan2.license_pool, loan2, None, None, loan2.license_pool.identifier + ) + # Fulfill and revoke. + [revoke, fulfill] = sorted(loan2_links, key=lambda x: x.rel or "") + assert revoke.href and "revoke_loan_or_hold" in revoke.href + assert "http://librarysimplified.org/terms/rel/revoke" == revoke.rel + assert fulfill.href and "fulfill" in fulfill.href + assert "http://opds-spec.org/acquisition" == fulfill.rel + + assert strftime(loan2.start) == fulfill.availability_since + assert strftime(loan2.end) == fulfill.availability_until + + # If a book is ready to be fulfilled, but the library has + # hidden all of its available content types, the fulfill link does + # not show up -- only the revoke link. + library = annotator_fixture.db.default_library() + settings = library_fixture.settings(library) + available_types = [ + lpdm.delivery_mechanism.content_type + for lpdm in loan2.license_pool.delivery_mechanisms + ] + settings.hidden_content_types = available_types + + # The list of hidden content types is stored in the Annotator + # constructor, so this particular test needs a fresh Annotator. + annotator_with_hidden_types = LibraryLoanAndHoldAnnotator( + None, None, annotator_fixture.db.default_library() + ) + loan2_links = annotator_with_hidden_types.acquisition_links( + loan2.license_pool, loan2, None, None, loan2.license_pool.identifier + ) + [revoke] = loan2_links + assert "http://librarysimplified.org/terms/rel/revoke" == revoke.rel + # Un-hide the content types so the test can continue. + settings.hidden_content_types = [] + + hold_links = annotator.acquisition_links( + hold.license_pool, None, hold, None, hold.license_pool.identifier + ) + # Borrow and revoke. + [revoke, borrow] = sorted(hold_links, key=lambda x: x.rel or "") + assert revoke.href and "revoke_loan_or_hold" in revoke.href + assert "http://librarysimplified.org/terms/rel/revoke" == revoke.rel + assert borrow.href and "borrow" in borrow.href + assert "http://opds-spec.org/acquisition/borrow" == borrow.rel + + work4_links = annotator.acquisition_links( + work4.license_pools[0], + None, + None, + None, + work4.license_pools[0].identifier, + ) + # Borrow only. + [borrow] = work4_links + assert borrow.href and "borrow" in borrow.href + assert "http://opds-spec.org/acquisition/borrow" == borrow.rel + + loan5_links = annotator.acquisition_links( + loan5.license_pool, loan5, None, None, loan5.license_pool.identifier + ) + # Fulfill and revoke. + [revoke, fulfill] = sorted(loan5_links, key=lambda x: x.rel or "") + assert revoke.href and "revoke_loan_or_hold" in revoke.href + assert "http://librarysimplified.org/terms/rel/revoke" == revoke.rel + assert fulfill.href and "fulfill" in fulfill.href + assert "http://opds-spec.org/acquisition" == fulfill.rel + + assert strftime(loan5.start) == fulfill.availability_since + # TODO: This currently fails, it should be uncommented when the CM 21 day loan bug is fixed + # assert loan5.end == availability.until + assert None == loan5.end + + # If patron authentication is turned off for the library, then + # only open-access links are displayed. + annotator.identifies_patrons = False + + [open_access] = annotator.acquisition_links( + loan1.license_pool, loan1, None, None, loan1.license_pool.identifier + ) + assert "http://opds-spec.org/acquisition/open-access" == open_access.rel + + # This may include links with the open-access relation for + # non-open-access works that are available without + # authentication. To get such link, you pass in a list of + # LicensePoolDeliveryMechanisms as + # `direct_fufillment_delivery_mechanisms`. + [lp4] = work4.license_pools + [lpdm4] = lp4.delivery_mechanisms + lpdm4.set_rights_status(RightsStatus.IN_COPYRIGHT) + [not_open_access] = annotator.acquisition_links( + lp4, + None, + None, + None, + lp4.identifier, + direct_fulfillment_delivery_mechanisms=[lpdm4], + ) + + # The link relation is OPDS 'open-access', which just means the + # book can be downloaded with no hassle. + assert "http://opds-spec.org/acquisition/open-access" == not_open_access.rel + + # The dcterms:rights attribute provides a more detailed + # explanation of the book's copyright status -- note that it's + # not "open access" in the typical sense. + rights = not_open_access.rights + assert RightsStatus.IN_COPYRIGHT == rights + + # Hold links are absent even when there are active holds in the + # database -- there is no way to distinguish one patron from + # another so the concept of a 'hold' is meaningless. + hold_links = annotator.acquisition_links( + hold.license_pool, None, hold, None, hold.license_pool.identifier + ) + assert [] == hold_links + + def test_acquisition_links_multiple_links( + self, + annotator_fixture: LibraryAnnotatorFixture, + library_fixture: LibraryFixture, + ): + annotator = LibraryLoanAndHoldAnnotator( + None, None, annotator_fixture.db.default_library() + ) + + # This book has two delivery mechanisms + work = annotator_fixture.db.work(with_license_pool=True) + [pool] = work.license_pools + [mech1] = pool.delivery_mechanisms + mech2 = pool.set_delivery_mechanism( + Representation.PDF_MEDIA_TYPE, + DeliveryMechanism.NO_DRM, + RightsStatus.IN_COPYRIGHT, + None, + ) + + # The vendor API for LicensePools of this type requires that a + # delivery mechanism be chosen at the point of borrowing. + class MockAPI: + SET_DELIVERY_MECHANISM_AT = BaseCirculationAPI.BORROW_STEP + + # This means that two different acquisition links will be + # generated -- one for each delivery mechanism. + links = annotator.acquisition_links( + pool, None, None, None, pool.identifier, mock_api=MockAPI() + ) + assert 2 == len(links) + + mech1_param = "mechanism_id=%s" % mech1.delivery_mechanism.id + mech2_param = "mechanism_id=%s" % mech2.delivery_mechanism.id + + # Instead of sorting, which may be wrong if the id is greater than 10 + # due to how double digits are sorted, extract the links associated + # with the expected delivery mechanism. + if links[0].href and mech1_param in links[0].href: + [mech1_link, mech2_link] = links + else: + [mech2_link, mech1_link] = links + + indirects = [] + for link in [mech1_link, mech2_link]: + # Both links should have the same subtags. + assert link.availability_status is not None + assert link.copies_total is not None + assert link.holds_total is not None + assert len(link.indirect_acquisitions) > 0 + indirects.append(link.indirect_acquisitions[0]) + + # The target of the top-level link is different. + assert mech1_link.href and mech1_param in mech1_link.href + assert mech2_link.href and mech2_param in mech2_link.href + + # So is the media type seen in the indirectAcquisition subtag. + [mech1_indirect, mech2_indirect] = indirects + + # The first delivery mechanism (created when the Work was created) + # uses Adobe DRM, so that shows up as the first indirect acquisition + # type. + assert mech1.delivery_mechanism.drm_scheme == mech1_indirect.type + + # The second delivery mechanism doesn't use DRM, so the content + # type shows up as the first (and only) indirect acquisition type. + assert mech2.delivery_mechanism.content_type == mech2_indirect.type + + # If we configure the library to hide one of the content types, + # we end up with only one link -- the one for the delivery + # mechanism that's not hidden. + library = annotator_fixture.db.default_library() + settings = library_fixture.settings(library) + settings.hidden_content_types = [mech1.delivery_mechanism.content_type] + annotator = LibraryLoanAndHoldAnnotator( + None, None, annotator_fixture.db.default_library() + ) + [link] = annotator.acquisition_links( + pool, None, None, None, pool.identifier, mock_api=MockAPI() + ) + assert ( + mech2.delivery_mechanism.content_type == link.indirect_acquisitions[0].type + ) diff --git a/tests/api/feed/test_loan_and_hold_annotator.py b/tests/api/feed/test_loan_and_hold_annotator.py new file mode 100644 index 0000000000..79df7ed502 --- /dev/null +++ b/tests/api/feed/test_loan_and_hold_annotator.py @@ -0,0 +1,287 @@ +from unittest.mock import MagicMock, patch + +from api.app import app +from api.problem_details import NOT_FOUND_ON_REMOTE +from core.classifier import ( # type: ignore[attr-defined] + Classifier, + Fantasy, + Urban_Fantasy, +) +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.loan_and_hold import LibraryLoanAndHoldAnnotator +from core.feed.types import WorkEntry, WorkEntryData +from core.lane import WorkList +from core.model import ExternalIntegration, get_one +from core.model.constants import EditionConstants, LinkRelations +from core.model.licensing import LicensePool +from core.model.patron import Loan +from tests.fixtures.database import DatabaseTransactionFixture + + +class TestLibraryLoanAndHoldAnnotator: + def test_single_item_feed(self, db: DatabaseTransactionFixture): + # Test the generation of single-item OPDS feeds for loans (with and + # without fulfillment) and holds. + class MockAnnotator(LibraryLoanAndHoldAnnotator): + def url_for(self, controller, **kwargs): + self.url_for_called_with = (controller, kwargs) + return "a URL" + + def mock_single_entry(work, annotator, *args, **kwargs): + annotator._single_entry_response_called_with = ( + (work, annotator) + args, + kwargs, + ) + w = WorkEntry( + work=work, + license_pool=work.active_license_pool(), + edition=work.presentation_edition, + identifier=work.presentation_edition.primary_identifier, + ) + w.computed = WorkEntryData() + return w + + def test_annotator(item, fulfillment=None): + # Call MockAnnotator.single_item_feed with certain arguments + # and make some general assertions about the return value. + circulation = object() + test_mode = object() + feed_class = object() + annotator = MockAnnotator(MagicMock(), None, db.default_library()) + + with patch.object( + OPDSAcquisitionFeed, "single_entry", new=mock_single_entry + ): + result = OPDSAcquisitionFeed.single_entry_loans_feed( + MagicMock(), + item, + annotator, + fulfillment=fulfillment, + ) + + assert db.default_library() == annotator.library + + # Now let's see what we did with it after calling its + # constructor. + + # The return value of that was the string "a URL". We then + # passed that into _single_entry_response, along with + # `item` and a number of arguments that we made up. + response_call = annotator._single_entry_response_called_with + (_work, _annotator), kwargs = response_call + assert work == _work + assert annotator == _annotator + + # Return the MockAnnotator for further examination. + return annotator + + # Now we're going to call test_annotator a couple times in + # different situations. + work = db.work(with_license_pool=True) + [pool] = work.license_pools + patron = db.patron() + loan, ignore = pool.loan_to(patron) + + # First, let's ask for a single-item feed for a loan. + annotator = test_annotator(loan) + + # Everything tested by test_annotator happened, but _also_, + # when the annotator was created, the Loan was stored in + # active_loans_by_work. + assert {work: loan} == annotator.active_loans_by_work + + # Since we passed in a loan rather than a hold, + # active_holds_by_work is empty. + assert {} == annotator.active_holds_by_work + + # Since we didn't pass in a fulfillment for the loan, + # active_fulfillments_by_work is empty. + assert {} == annotator.active_fulfillments_by_work + + # Now try it again, but give the loan a fulfillment. + fulfillment = object() + annotator = test_annotator(loan, fulfillment) + assert {work: loan} == annotator.active_loans_by_work + assert {work: fulfillment} == annotator.active_fulfillments_by_work + + # Finally, try it with a hold. + hold, ignore = pool.on_hold_to(patron) + annotator = test_annotator(hold) + assert {work: hold} == annotator.active_holds_by_work + assert {} == annotator.active_loans_by_work + assert {} == annotator.active_fulfillments_by_work + + def test_single_item_feed_without_work(self, db: DatabaseTransactionFixture): + """If a licensepool has no work or edition the single_item_feed mustn't raise an exception""" + mock = MagicMock() + # A loan without a pool + annotator = LibraryLoanAndHoldAnnotator(mock, None, db.default_library()) + loan = Loan() + loan.patron = db.patron() + not_found_result = OPDSAcquisitionFeed.single_entry_loans_feed( + mock, + loan, + annotator, + ) + assert not_found_result == NOT_FOUND_ON_REMOTE + + work = db.work(with_license_pool=True) + pool = get_one(db.session, LicensePool, work_id=work.id) + assert isinstance(pool, LicensePool) + # Pool with no work, and the presentation edition has no work either + pool.work_id = None + work.presentation_edition_id = None + db.session.commit() + assert ( + OPDSAcquisitionFeed.single_entry_loans_feed( + mock, + pool, + annotator, + ) + == NOT_FOUND_ON_REMOTE + ) + + # pool with no work and no presentation edition + pool.presentation_edition_id = None + db.session.commit() + assert ( + OPDSAcquisitionFeed.single_entry_loans_feed( + mock, + pool, + annotator, + ) + == NOT_FOUND_ON_REMOTE + ) + + def test_choose_best_hold_for_work(self, db: DatabaseTransactionFixture): + # First create two license pools for the same work so we could create two holds for the same work. + patron = db.patron() + + coll_1 = db.collection(name="Collection 1") + coll_2 = db.collection(name="Collection 2") + + work = db.work() + + pool_1 = db.licensepool( + edition=work.presentation_edition, open_access=False, collection=coll_1 + ) + pool_2 = db.licensepool( + edition=work.presentation_edition, open_access=False, collection=coll_2 + ) + + hold_1, _ = pool_1.on_hold_to(patron) + hold_2, _ = pool_2.on_hold_to(patron) + + # When there is no licenses_owned/available on one license pool the LibraryLoanAndHoldAnnotator should choose + # hold associated with the other license pool. + pool_1.licenses_owned = 0 + pool_1.licenses_available = 0 + + assert hold_2 == LibraryLoanAndHoldAnnotator.choose_best_hold_for_work( + [hold_1, hold_2] + ) + + # Now we have different number of licenses owned across two LPs and the same hold position. + # Hold associated with LP with more owned licenses will be chosen as best. + pool_1.licenses_owned = 2 + + pool_2.licenses_owned = 3 + pool_2.licenses_available = 0 + + hold_1.position = 7 + hold_2.position = 7 + + assert hold_2 == LibraryLoanAndHoldAnnotator.choose_best_hold_for_work( + [hold_1, hold_2] + ) + + def test_annotate_work_entry(self, db: DatabaseTransactionFixture): + library = db.default_library() + patron = db.patron() + identifier = db.identifier() + lane = WorkList() + lane.initialize( + library, + ) + annotator = LibraryLoanAndHoldAnnotator(None, lane, library, patron) + feed = OPDSAcquisitionFeed("title", "url", [], annotator) + + # Annotate time tracking + opds_for_distributors = db.collection( + protocol=ExternalIntegration.OPDS_FOR_DISTRIBUTORS + ) + work = db.work(with_license_pool=True, collection=opds_for_distributors) + edition = work.presentation_edition + edition.medium = EditionConstants.AUDIO_MEDIUM + edition.primary_identifier = identifier + loan, _ = work.active_license_pool().loan_to(patron) + annotator.active_loans_by_work = {work: loan} + + with app.test_request_context("/") as request: + request.library = library # type: ignore [attr-defined] + entry = feed.single_entry(work, annotator) + assert isinstance(entry, WorkEntry) + assert entry and entry.computed is not None + time_tracking_links = list( + filter( + lambda l: l.rel == LinkRelations.TIME_TRACKING, + entry.computed.other_links, + ) + ) + assert len(time_tracking_links) == 1 + assert time_tracking_links[0].href == annotator.url_for( + "track_playtime_events", + identifier_type=identifier.type, + identifier=identifier.identifier, + library_short_name=annotator.library.short_name, + collection_id=opds_for_distributors.id, + _external=True, + ) + + # No active loan means no tracking link + annotator.active_loans_by_work = {} + entry = feed.single_entry(work, annotator) + assert isinstance(entry, WorkEntry) + assert entry and entry.computed is not None + + time_tracking_links = list( + filter( + lambda l: l.rel == LinkRelations.TIME_TRACKING, + entry.computed.other_links, + ) + ) + assert len(time_tracking_links) == 0 + + # Add the loan back in + annotator.active_loans_by_work = {work: loan} + + # Book mediums don't get time tracking + edition.medium = EditionConstants.BOOK_MEDIUM + entry = feed.single_entry(work, annotator) + assert isinstance(entry, WorkEntry) + assert entry and entry.computed is not None + + time_tracking_links = list( + filter( + lambda l: l.rel == LinkRelations.TIME_TRACKING, + entry.computed.other_links, + ) + ) + assert len(time_tracking_links) == 0 + + # Non OPDS for distributor works do not get links either + work = db.work(with_license_pool=True) + edition = work.presentation_edition + edition.medium = EditionConstants.AUDIO_MEDIUM + + entry = feed.single_entry(work, annotator) + assert isinstance(entry, WorkEntry) + assert entry and entry.computed is not None + + time_tracking_links = list( + filter( + lambda l: l.rel == LinkRelations.TIME_TRACKING, + entry.computed.other_links, + ) + ) + assert len(time_tracking_links) == 0 diff --git a/tests/api/feed/test_opds2_serializer.py b/tests/api/feed/test_opds2_serializer.py new file mode 100644 index 0000000000..2b2bfcdf68 --- /dev/null +++ b/tests/api/feed/test_opds2_serializer.py @@ -0,0 +1,215 @@ +import json + +from core.feed.serializer.opds2 import OPDS2Serializer +from core.feed.types import ( + Acquisition, + Author, + FeedData, + FeedEntryType, + IndirectAcquisition, + Link, + WorkEntry, + WorkEntryData, +) +from core.model.edition import Edition +from core.model.identifier import Identifier +from core.model.work import Work +from core.util.opds_writer import OPDSMessage + + +class TestOPDS2Serializer: + def test_serialize_feed(self): + feed = FeedData( + metadata=dict( + items_per_page=FeedEntryType(text="20"), + title=FeedEntryType(text="Title"), + ) + ) + w = WorkEntry( + work=Work(), + edition=Edition(), + identifier=Identifier(), + ) + w.computed = WorkEntryData(identifier="identifier", pwid="permanent-id") + feed.entries = [w] + feed.links = [Link(href="http://link", rel="link-rel")] + feed.facet_links = [ + Link.create( + href="http://facet-link", rel="facet-rel", facetGroup="FacetGroup" + ) + ] + + serialized = OPDS2Serializer().serialize_feed(feed) + result = json.loads(serialized) + + assert result["metadata"]["title"] == "Title" + assert result["metadata"]["itemsPerPage"] == 20 + + assert len(result["publications"]) == 1 + assert result["publications"][0] == dict( + metadata={"identifier": "identifier"}, images=[], links=[] + ) + + assert len(result["links"]) == 1 + assert result["links"][0] == dict(href="http://link", rel="link-rel") + + assert len(result["facets"]) == 1 + assert result["facets"][0] == dict( + metadata={"title": "FacetGroup"}, + links=[{"href": "http://facet-link", "rel": "facet-rel"}], + ) + + def test_serialize_work_entry(self): + data = WorkEntryData( + additionalType="type", + title=FeedEntryType(text="The Title"), + sort_title=FeedEntryType(text="Title, The"), + subtitle=FeedEntryType(text="Sub Title"), + identifier="urn:id", + language=FeedEntryType(text="de"), + updated=FeedEntryType(text="2022-02-02"), + published=FeedEntryType(text="2020-02-02"), + summary=FeedEntryType(text="Summary"), + publisher=FeedEntryType(text="Publisher"), + imprint=FeedEntryType(text="Imprint"), + categories=[ + FeedEntryType.create(scheme="scheme", label="label"), + ], + series=FeedEntryType.create(name="Series", position="3"), + image_links=[Link(href="http://image", rel="image-rel")], + acquisition_links=[ + Acquisition(href="http://acquisition", rel="acquisition-rel") + ], + other_links=[Link(href="http://link", rel="rel")], + ) + + serializer = OPDS2Serializer() + + entry = serializer.serialize_work_entry(data) + metadata = entry["metadata"] + + assert metadata["@type"] == data.additionalType + assert metadata["title"] == data.title.text + assert metadata["sortAs"] == data.sort_title.text + assert metadata["subtitle"] == data.subtitle.text + assert metadata["identifier"] == data.identifier + assert metadata["language"] == data.language.text + assert metadata["modified"] == data.updated.text + assert metadata["published"] == data.published.text + assert metadata["description"] == data.summary.text + assert metadata["publisher"] == dict(name=data.publisher.text) + assert metadata["imprint"] == dict(name=data.imprint.text) + assert metadata["subject"] == [ + dict(scheme="scheme", name="label", sortAs="label") + ] + assert metadata["belongsTo"] == dict(name="Series", position=3) + + assert entry["links"] == [ + dict(href="http://link", rel="rel"), + dict(href="http://acquisition", rel="acquisition-rel"), + ] + assert entry["images"] == [dict(href="http://image", rel="image-rel")] + + # Test the different author types + data = WorkEntryData( + authors=[Author(name="author1"), Author(name="author2")], + contributors=[ + Author(name="translator", role="trl"), + Author(name="editor", role="edt"), + Author(name="artist", role="art"), + Author(name="illustrator", role="ill"), + Author(name="letterer", role="ctb"), + Author(name="penciller", role="ctb"), + Author(name="colorist", role="clr"), + Author(name="inker", role="ctb"), + Author(name="narrator", role="nrt"), + Author(name="narrator2", role="nrt"), + ], + ) + + entry = serializer.serialize_work_entry(data) + metadata = entry["metadata"] + # Only the first author is considered + assert metadata["author"] == dict(name="author1") + # Of the allowed roles + assert metadata["translator"] == dict(name="translator") + assert metadata["editor"] == dict(name="editor") + assert metadata["artist"] == dict(name="artist") + assert metadata["illustrator"] == dict(name="illustrator") + assert metadata["colorist"] == dict(name="colorist") + # Of letterer, penciller, and inker, only inker is used, since the marc roles overlap + assert metadata["inker"] == dict(name="inker") + # Of repeated roles, only the last entry is picked + assert metadata["narrator"] == dict(name="narrator2") + + def test__serialize_acquisition_link(self): + serializer = OPDS2Serializer() + acquisition = Acquisition( + href="http://acquisition", + rel="acquisition", + availability_status="available", + availability_since="2022-02-02", + availability_until="2222-02-02", + indirect_acquisitions=[ + IndirectAcquisition( + type="indirect1", + children=[ + IndirectAcquisition(type="indirect1-1"), + IndirectAcquisition(type="indirect1-2"), + ], + ), + ], + ) + + result = serializer._serialize_acquisition_link(acquisition) + + assert result["href"] == acquisition.href + assert result["rel"] == acquisition.rel + assert result["properties"] == dict( + availability={ + "since": "2022-02-02", + "until": "2222-02-02", + "state": "available", + }, + indirectAcquisition=[ + { + "type": "indirect1", + "child": [{"type": "indirect1-1"}, {"type": "indirect1-2"}], + } + ], + ) + + # Test availability states + acquisition = Acquisition( + href="http://hold", + rel="hold", + is_hold=True, + availability_status="available", + ) + result = serializer._serialize_acquisition_link(acquisition) + assert result["properties"]["availability"]["state"] == "reserved" + + acquisition = Acquisition( + href="http://loan", + rel="loan", + is_loan=True, + availability_status="available", + ) + result = serializer._serialize_acquisition_link(acquisition) + assert result["properties"]["availability"]["state"] == "ready" + + def test__serialize_contributor(self): + author = Author( + name="Author", + sort_name="Author,", + link=Link(href="http://author", rel="contributor", title="Delete me!"), + ) + result = OPDS2Serializer()._serialize_contributor(author) + assert result["name"] == "Author" + assert result["sortAs"] == "Author," + assert result["links"] == [{"href": "http://author", "rel": "contributor"}] + + def test_serialize_opds_message(self): + assert OPDS2Serializer().serialize_opds_message( + OPDSMessage("URN", 200, "Description") + ) == dict(urn="URN", description="Description") diff --git a/tests/api/feed/test_opds_acquisition_feed.py b/tests/api/feed/test_opds_acquisition_feed.py new file mode 100644 index 0000000000..3f6a098713 --- /dev/null +++ b/tests/api/feed/test_opds_acquisition_feed.py @@ -0,0 +1,1454 @@ +import datetime +import logging +from collections import defaultdict +from typing import Any, Callable, Generator, List, Type +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy.orm import Session +from werkzeug.datastructures import MIMEAccept + +from core.entrypoint import ( + AudiobooksEntryPoint, + EbooksEntryPoint, + EntryPoint, + EverythingEntryPoint, + MediumEntryPoint, +) +from core.external_search import MockExternalSearchIndex +from core.facets import FacetConstants +from core.feed.acquisition import LookupAcquisitionFeed, OPDSAcquisitionFeed +from core.feed.annotator.base import Annotator +from core.feed.annotator.circulation import ( + AcquisitionHelper, + CirculationManagerAnnotator, + LibraryAnnotator, +) +from core.feed.annotator.loan_and_hold import LibraryLoanAndHoldAnnotator +from core.feed.annotator.verbose import VerboseAnnotator +from core.feed.navigation import NavigationFeed +from core.feed.opds import BaseOPDSFeed +from core.feed.types import FeedData, Link, WorkEntry, WorkEntryData +from core.lane import Facets, FeaturedFacets, Lane, Pagination, SearchFacets, WorkList +from core.model import DeliveryMechanism, Representation +from core.model.constants import LinkRelations +from core.opds import MockUnfulfillableAnnotator +from core.util.datetime_helpers import utc_now +from core.util.flask_util import OPDSEntryResponse, OPDSFeedResponse +from core.util.opds_writer import OPDSFeed, OPDSMessage +from tests.api.feed.fixtures import PatchedUrlFor, patch_url_for # noqa +from tests.fixtures.database import DatabaseTransactionFixture +from tests.fixtures.search import ExternalSearchPatchFixture + + +class TestOPDSFeedProtocol: + def test_entry_as_response(self, db: DatabaseTransactionFixture): + work = db.work() + entry = WorkEntry( + work=work, + edition=work.presentation_edition, + identifier=work.presentation_edition.primary_identifier, + ) + + with pytest.raises(ValueError) as raised: + BaseOPDSFeed.entry_as_response(entry) + assert str(raised.value) == "Entry data has not been generated" + + entry.computed = WorkEntryData() + + response = BaseOPDSFeed.entry_as_response(entry) + assert isinstance(response, OPDSEntryResponse) + # default content type is XML + assert response.content_type == OPDSEntryResponse().content_type + + # Specifically asking for a json type + response = BaseOPDSFeed.entry_as_response( + entry, mime_types=MIMEAccept([("application/opds+json", 0.9)]) + ) + assert isinstance(response, OPDSEntryResponse) + assert response.content_type == "application/opds+json" + + response = BaseOPDSFeed.entry_as_response( + OPDSMessage("URN", 204, "Test OPDS Message") + ) + assert isinstance(response, OPDSEntryResponse) + assert response.status_code == 204 + assert ( + b"Test OPDS Message" + in response.data + ) + + response = BaseOPDSFeed.entry_as_response( + OPDSMessage("URN", 204, "Test OPDS Message"), + mime_types=MIMEAccept([("application/opds+json", 1)]), + ) + assert isinstance(response, OPDSEntryResponse) + assert response.status_code == 204 + assert dict(description="Test OPDS Message", urn="URN") == response.json + + +class MockAnnotator(CirculationManagerAnnotator): + def __init__(self): + self.lanes_by_work = defaultdict(list) + + @classmethod + def lane_url(cls, lane): + if lane and lane.has_visible_children: + return cls.groups_url(lane) + elif lane: + return cls.feed_url(lane) + else: + return "" + + @classmethod + def feed_url(cls, lane, facets=None, pagination=None): + if isinstance(lane, Lane): + base = "http://%s/" % lane.url_name + else: + base = "http://%s/" % lane.display_name + sep = "?" + if facets: + base += sep + facets.query_string + sep = "&" + if pagination: + base += sep + pagination.query_string + return base + + @classmethod + def search_url(cls, lane, query, pagination, facets=None): + if isinstance(lane, Lane): + base = "http://%s/" % lane.url_name + else: + base = "http://%s/" % lane.display_name + sep = "?" + if pagination: + base += sep + pagination.query_string + sep = "&" + if facets: + facet_query_string = facets.query_string + if facet_query_string: + base += sep + facet_query_string + return base + + @classmethod + def groups_url(cls, lane, facets=None): + if lane and isinstance(lane, Lane): + identifier = lane.id + else: + identifier = "" + if facets: + facet_string = "?" + facets.query_string + else: + facet_string = "" + + return f"http://groups/{identifier}{facet_string}" + + @classmethod + def default_lane_url(cls): + return cls.groups_url(None) + + @classmethod + def facet_url(cls, facets): + return "http://facet/" + "&".join( + [f"{k}={v}" for k, v in sorted(facets.items())] + ) + + @classmethod + def navigation_url(cls, lane): + if lane and isinstance(lane, Lane): + identifier = lane.id + else: + identifier = "" + return "http://navigation/%s" % identifier + + @classmethod + def top_level_title(cls): + return "Test Top Level Title" + + +class TestOPDSAcquisitionFeed: + def test_page( + self, + db, + external_search_patch_fixture: ExternalSearchPatchFixture, + ): + session = db.session + + # Verify that AcquisitionFeed.page() returns an appropriate OPDSFeedResponse + + wl = WorkList() + wl.initialize(db.default_library()) + private = object() + response = OPDSAcquisitionFeed.page( + session, + "feed title", + "url", + wl, + CirculationManagerAnnotator(None), + None, + None, + None, + ).as_response(max_age=10, private=private) + + # The result is an OPDSFeedResponse. The 'private' argument, + # unused by page(), was passed along into the constructor. + assert isinstance(response, OPDSFeedResponse) + assert 10 == response.max_age + assert private == response.private + + assert "feed title" in str(response) + + def test_as_response(self, db: DatabaseTransactionFixture): + session = db.session + + # Verify the ability to convert an AcquisitionFeed object to an + # OPDSFeedResponse containing the feed. + feed = OPDSAcquisitionFeed( + "feed title", + "http://url/", + [], + CirculationManagerAnnotator(None), + ) + feed.generate_feed() + + # Some other piece of code set expectations for how this feed should + # be cached. + response = feed.as_response(max_age=101, private=False) + assert 200 == response.status_code + + # We get an OPDSFeedResponse containing the feed in its + # entity-body. + assert isinstance(response, OPDSFeedResponse) + assert "feed title" in str(response) + + # The caching expectations are respected. + assert 101 == response.max_age + assert False == response.private + + def test_as_error_response(self, db: DatabaseTransactionFixture): + session = db.session + + # Verify the ability to convert an AcquisitionFeed object to an + # OPDSFeedResponse that is to be treated as an error message. + feed = OPDSAcquisitionFeed( + "feed title", + "http://url/", + [], + CirculationManagerAnnotator(None), + ) + feed.generate_feed() + + # Some other piece of code set expectations for how this feed should + # be cached. + kwargs = dict(max_age=101, private=False) + + # But we know that something has gone wrong and the feed is + # being served as an error message. + response = feed.as_error_response(**kwargs) + assert isinstance(response, OPDSFeedResponse) + + # The content of the feed is unchanged. + assert 200 == response.status_code + assert "feed title" in str(response) + + # But the max_age and private settings have been overridden. + assert 0 == response.max_age + assert True == response.private + + def test_add_entrypoint_links(self): + """Verify that add_entrypoint_links calls _entrypoint_link + on every EntryPoint passed in. + """ + + class Mock: + attrs = dict(href="the response") + + def __init__(self): + self.calls = [] + + def __call__(self, *args): + self.calls.append(args) + return Link(**self.attrs) + + mock = Mock() + old_entrypoint_link = OPDSAcquisitionFeed._entrypoint_link + OPDSAcquisitionFeed._entrypoint_link = mock + + feed = FeedData() + entrypoints = [AudiobooksEntryPoint, EbooksEntryPoint] + url_generator = object() + OPDSAcquisitionFeed.add_entrypoint_links( + feed, url_generator, entrypoints, EbooksEntryPoint, "Some entry points" + ) + + # Two different calls were made to the mock method. + c1, c2 = mock.calls + + # The first entry point is not selected. + assert c1 == ( + url_generator, + AudiobooksEntryPoint, + EbooksEntryPoint, + True, + "Some entry points", + ) + # The second one is selected. + assert c2 == ( + url_generator, + EbooksEntryPoint, + EbooksEntryPoint, + False, + "Some entry points", + ) + + # Two identical tags were added to the tag, one + # for each call to the mock method. + l1, l2 = feed.links + for l in l1, l2: + assert mock.attrs == l.link_attribs() + OPDSAcquisitionFeed._entrypoint_link = old_entrypoint_link + + # If there is only one facet in the facet group, no links are + # added. + feed = FeedData() + mock.calls = [] + entrypoints = [EbooksEntryPoint] + OPDSAcquisitionFeed.add_entrypoint_links( + feed, url_generator, entrypoints, EbooksEntryPoint, "Some entry points" + ) + assert [] == mock.calls + + def test_entrypoint_link(self): + """Test the _entrypoint_link method's ability to create + attributes for tags. + """ + m = OPDSAcquisitionFeed._entrypoint_link + + def g(entrypoint): + """A mock URL generator.""" + return "%s" % (entrypoint.INTERNAL_NAME) + + # If the entry point is not registered, None is returned. + assert None == m(g, object(), object(), True, "group") + + # Now make a real set of link attributes. + l = m(g, AudiobooksEntryPoint, AudiobooksEntryPoint, False, "Grupe") + + # The link is identified as belonging to an entry point-type + # facet group. + assert l.rel == LinkRelations.FACET_REL + assert getattr(l, "facetGroupType") == FacetConstants.ENTRY_POINT_REL + assert "Grupe" == getattr(l, "facetGroup") + + # This facet is the active one in the group. + assert "true" == getattr(l, "activeFacet") + + # The URL generator was invoked to create the href. + assert l.href == g(AudiobooksEntryPoint) + + # The facet title identifies it as a way to look at audiobooks. + assert EntryPoint.DISPLAY_TITLES[AudiobooksEntryPoint] == l.title + + # Now try some variants. + + # Here, the entry point is the default one. + l = m(g, AudiobooksEntryPoint, AudiobooksEntryPoint, True, "Grupe") + + # This may affect the URL generated for the facet link. + assert l.href == g(AudiobooksEntryPoint) + + # Here, the entry point for which we're generating the link is + # not the selected one -- EbooksEntryPoint is. + l = m(g, AudiobooksEntryPoint, EbooksEntryPoint, True, "Grupe") + + # This means the 'activeFacet' attribute is not present. + assert getattr(l, "activeFacet", None) == None + + def test_license_tags_no_loan_or_hold(self, db: DatabaseTransactionFixture): + edition, pool = db.edition(with_license_pool=True) + tags = AcquisitionHelper.license_tags(pool, None, None) + assert ( + dict( + availability_status="available", + holds_total="0", + copies_total="1", + copies_available="1", + ) + == tags + ) + + def test_license_tags_hold_position(self, db: DatabaseTransactionFixture): + # When a book is placed on hold, it typically takes a while + # for the LicensePool to be updated with the new number of + # holds. This test verifies the normal and exceptional + # behavior used to generate the opds:holds tag in different + # scenarios. + edition, pool = db.edition(with_license_pool=True) + patron = db.patron() + + # If the patron's hold position is less than the total number + # of holds+reserves, that total is used as opds:total. + pool.patrons_in_hold_queue = 3 + hold, is_new = pool.on_hold_to(patron, position=1) + + tags = AcquisitionHelper.license_tags(pool, None, hold) + assert tags is not None + assert "1" == tags["holds_position"] + assert "3" == tags["holds_total"] + + # If the patron's hold position is missing, we assume they + # are last in the list. + hold.position = None + tags = AcquisitionHelper.license_tags(pool, None, hold) + assert tags is not None + assert "3" == tags["holds_position"] + assert "3" == tags["holds_total"] + + # If the patron's current hold position is greater than the + # total recorded number of holds+reserves, their position will + # be used as the value of opds:total. + hold.position = 5 + tags = AcquisitionHelper.license_tags(pool, None, hold) + assert tags is not None + assert "5" == tags["holds_position"] + assert "5" == tags["holds_total"] + + # A patron earlier in the holds queue may see a different + # total number of holds, but that's fine -- it doesn't matter + # very much to that person the precise number of people behind + # them in the queue. + hold.position = 4 + tags = AcquisitionHelper.license_tags(pool, None, hold) + assert tags is not None + assert "4" == tags["holds_position"] + assert "4" == tags["holds_total"] + + # If the patron's hold position is zero (because the book is + # reserved to them), we do not represent them as having a hold + # position (so no opds:position), but they still count towards + # opds:total in the case where the LicensePool's information + # is out of date. + hold.position = 0 + pool.patrons_in_hold_queue = 0 + tags = AcquisitionHelper.license_tags(pool, None, hold) + assert tags is not None + assert "holds_position" not in tags + assert "1" == tags["holds_total"] + + def test_license_tags_show_unlimited_access_books( + self, db: DatabaseTransactionFixture + ): + # Arrange + edition, pool = db.edition(with_license_pool=True) + pool.open_access = False + pool.unlimited_access = True + + # Act + tags = AcquisitionHelper.license_tags(pool, None, None) + + # Assert + assert tags is not None + assert 1 == len(tags.keys()) + assert tags["availability_status"] == "available" + + def test_unlimited_access_pool_loan(self, db: DatabaseTransactionFixture): + patron = db.patron() + work = db.work(unlimited_access=True, with_license_pool=True) + pool = work.active_license_pool() + loan, _ = pool.loan_to(patron) + tags = AcquisitionHelper.license_tags(pool, loan, None) + + assert tags is not None + assert "availability_since" in tags + assert "availability_until" not in tags + + def test_single_entry(self, db: DatabaseTransactionFixture): + session = db.session + + # Here's a Work with two LicensePools. + work = db.work(with_open_access_download=True) + original_pool = work.license_pools[0] + edition, new_pool = db.edition( + with_license_pool=True, with_open_access_download=True + ) + work.license_pools.append(new_pool) + + # The presentation edition of the Work is associated with + # the first LicensePool added to it. + assert work.presentation_edition == original_pool.presentation_edition + + # This is the edition used when we create an tag for + # this Work. + private = object() + entry = OPDSAcquisitionFeed.single_entry( + work, + Annotator(), + ) + assert isinstance(entry, WorkEntry) + assert entry.computed is not None + assert entry.computed.title is not None + + assert new_pool.presentation_edition.title != entry.computed.title.text + assert original_pool.presentation_edition.title == entry.computed.title.text + + # If the edition was issued before 1980, no datetime formatting error + # is raised. + work.simple_opds_entry = work.verbose_opds_entry = None + five_hundred_years = datetime.timedelta(days=(500 * 365)) + work.presentation_edition.issued = utc_now() - five_hundred_years + + entry = OPDSAcquisitionFeed.single_entry(work, Annotator()) + assert isinstance(entry, WorkEntry) + assert entry.computed is not None + assert entry.computed.issued is not None + + assert work.presentation_edition.issued == entry.computed.issued + + def test_error_when_work_has_no_identifier(self, db: DatabaseTransactionFixture): + session = db.session + + # We cannot create an OPDS entry for a Work that cannot be associated + # with an Identifier. + work = db.work(title="Hello, World!", with_license_pool=True) + work.license_pools[0].identifier = None + work.presentation_edition.primary_identifier = None + entry = OPDSAcquisitionFeed.single_entry(work, Annotator()) + assert entry == None + + def test_error_when_work_has_no_licensepool(self, db: DatabaseTransactionFixture): + session = db.session + + work = db.work() + entry = OPDSAcquisitionFeed.single_entry(work, Annotator()) + expect = OPDSAcquisitionFeed.error_message( + work.presentation_edition.primary_identifier, + 403, + "I've heard about this work but have no active licenses for it.", + ) + assert expect == entry + + def test_error_when_work_has_no_presentation_edition( + self, db: DatabaseTransactionFixture + ): + session = db.session + + """We cannot create an OPDS entry (or even an error message) for a + Work that is disconnected from any Identifiers. + """ + work = db.work(title="Hello, World!", with_license_pool=True) + work.license_pools[0].presentation_edition = None + work.presentation_edition = None + entry = OPDSAcquisitionFeed.single_entry(work, Annotator()) + assert None == entry + + def test_exception_during_entry_creation_is_not_reraised( + self, db: DatabaseTransactionFixture + ): + # This feed will raise an exception whenever it's asked + # to create an entry. + class DoomedFeed(OPDSAcquisitionFeed): + @classmethod + def _create_entry(cls, *args, **kwargs): + raise Exception("I'm doomed!") + + work = db.work(with_open_access_download=True) + + # But calling create_entry() doesn't raise an exception, it + # just returns None. + entry = DoomedFeed.single_entry(work, Annotator()) + assert entry == None + + def test_unfilfullable_work(self, db: DatabaseTransactionFixture): + work = db.work(with_open_access_download=True) + [pool] = work.license_pools + response = OPDSAcquisitionFeed.single_entry( + work, + MockUnfulfillableAnnotator(), # type: ignore[arg-type] + ) + assert isinstance(response, OPDSMessage) + expect = OPDSAcquisitionFeed.error_message( + pool.identifier, + 403, + "I know about this work but can offer no way of fulfilling it.", + ) + + assert str(expect) == str(response) + + def test_format_types(self, db: DatabaseTransactionFixture): + session = db.session + + m = AcquisitionHelper.format_types + + epub_no_drm, ignore = DeliveryMechanism.lookup( + session, Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.NO_DRM + ) + assert [Representation.EPUB_MEDIA_TYPE] == m(epub_no_drm) + + epub_adobe_drm, ignore = DeliveryMechanism.lookup( + session, Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.ADOBE_DRM + ) + assert [DeliveryMechanism.ADOBE_DRM, Representation.EPUB_MEDIA_TYPE] == m( + epub_adobe_drm + ) + + overdrive_streaming_text, ignore = DeliveryMechanism.lookup( + session, + DeliveryMechanism.STREAMING_TEXT_CONTENT_TYPE, + DeliveryMechanism.OVERDRIVE_DRM, + ) + assert [ + OPDSFeed.ENTRY_TYPE, + Representation.TEXT_HTML_MEDIA_TYPE + DeliveryMechanism.STREAMING_PROFILE, + ] == m(overdrive_streaming_text) + + audiobook_drm, ignore = DeliveryMechanism.lookup( + session, + Representation.AUDIOBOOK_MANIFEST_MEDIA_TYPE, + DeliveryMechanism.FEEDBOOKS_AUDIOBOOK_DRM, + ) + + assert [ + Representation.AUDIOBOOK_MANIFEST_MEDIA_TYPE + + DeliveryMechanism.FEEDBOOKS_AUDIOBOOK_PROFILE + ] == m(audiobook_drm) + + # Test a case where there is a DRM scheme but no underlying + # content type. + findaway_manifest, ignore = DeliveryMechanism.lookup( + session, DeliveryMechanism.FINDAWAY_DRM, None + ) + assert [DeliveryMechanism.FINDAWAY_DRM] == m(findaway_manifest) + + def test_add_breadcrumbs(self, db: DatabaseTransactionFixture): + session = db.session + _db = session + + def getElementChildren(feed): + f = feed.feed[0] + children = f + return children + + class MockFeed(OPDSAcquisitionFeed): + def __init__(self): + super().__init__("", "", [], MockAnnotator()) + self.feed = [] + + lane = db.lane(display_name="lane") + sublane = db.lane(parent=lane, display_name="sublane") + subsublane = db.lane(parent=sublane, display_name="subsublane") + subsubsublane = db.lane(parent=subsublane, display_name="subsubsublane") + + top_level = object() + ep = AudiobooksEntryPoint + + def assert_breadcrumbs(expect_breadcrumbs_for, lane, **add_breadcrumbs_kwargs): + # Create breadcrumbs leading up to `lane` and verify that + # there is a breadcrumb for everything in + # `expect_breadcrumbs_for` -- Lanes, EntryPoints, and the + # top-level lane. Verify that the titles and URLs of the + # breadcrumbs match what we expect. + # + # For easier reading, all assertions in this test are + # written as calls to this function. + feed = MockFeed() + annotator = MockAnnotator() + + feed.add_breadcrumbs(lane, **add_breadcrumbs_kwargs) + + if not expect_breadcrumbs_for: + # We are expecting no breadcrumbs at all; + # nothing should have been added to the feed. + assert [] == feed.feed + return + + # At this point we expect at least one breadcrumb. + crumbs = feed._feed.breadcrumbs + + entrypoint_selected = False + entrypoint_query = "?entrypoint=" + + # First, compare the titles of the breadcrumbs to what was + # passed in. This makes test writing much easier. + def title(x): + if x is top_level: + return annotator.top_level_title() + elif x is ep: + return x.INTERNAL_NAME + else: + return x.display_name + + expect_titles = [title(x) for x in expect_breadcrumbs_for] + actual_titles = [getattr(x, "title", None) for x in crumbs] + assert expect_titles == actual_titles + + # Now, compare the URLs of the breadcrumbs. This is + # trickier, mainly because the URLs change once an + # entrypoint is selected. + previous_breadcrumb_url = None + + for i, crumb in enumerate(crumbs): + expect = expect_breadcrumbs_for[i] + actual_url = crumb.href + + if expect is top_level: + # Breadcrumb for the library root. + expect_url = annotator.default_lane_url() + elif expect is ep: + # Breadcrumb for the entrypoint selection. + + # Beyond this point all URLs must propagate the + # selected entrypoint. + entrypoint_selected = True + entrypoint_query += expect.INTERNAL_NAME + + # The URL for this breadcrumb is the URL for the + # previous breadcrumb with the addition of the + # entrypoint selection query. + expect_url = previous_breadcrumb_url + entrypoint_query + else: + # Breadcrumb for a lane. + + # The breadcrumb URL is determined by the + # Annotator. + lane_url = annotator.lane_url(expect) + if entrypoint_selected: + # All breadcrumbs after the entrypoint selection + # must propagate the entrypoint. + expect_url = lane_url + entrypoint_query + else: + expect_url = lane_url + + logging.debug( + "%s: expect=%s actual=%s", expect_titles[i], expect_url, actual_url + ) + assert expect_url == actual_url + + # Keep track of the URL just used, in case the next + # breadcrumb is the same URL but with an entrypoint + # selection appended. + previous_breadcrumb_url = actual_url + + # That was a complicated method, but now our assertions + # are very easy to write and understand. + + # At the top level, there are no breadcrumbs whatsoever. + assert_breadcrumbs([], None) + + # It doesn't matter if an entrypoint is selected. + assert_breadcrumbs([], None, entrypoint=ep) + + # A lane with no entrypoint -- note that the breadcrumbs stop + # _before_ the lane in question. + assert_breadcrumbs([top_level], lane) + + # If you pass include_lane=True into add_breadcrumbs, the lane + # itself is included. + assert_breadcrumbs([top_level, lane], lane, include_lane=True) + + # A lane with an entrypoint selected + assert_breadcrumbs([top_level, ep], lane, entrypoint=ep) + assert_breadcrumbs( + [top_level, ep, lane], lane, entrypoint=ep, include_lane=True + ) + + # One lane level down. + assert_breadcrumbs([top_level, lane], sublane) + assert_breadcrumbs([top_level, ep, lane], sublane, entrypoint=ep) + assert_breadcrumbs( + [top_level, ep, lane, sublane], sublane, entrypoint=ep, include_lane=True + ) + + # Two lane levels down. + assert_breadcrumbs([top_level, lane, sublane], subsublane) + assert_breadcrumbs([top_level, ep, lane, sublane], subsublane, entrypoint=ep) + + # Three lane levels down. + assert_breadcrumbs( + [top_level, lane, sublane, subsublane], + subsubsublane, + ) + + assert_breadcrumbs( + [top_level, ep, lane, sublane, subsublane], subsubsublane, entrypoint=ep + ) + + # Make the sublane a root lane for a certain patron type, and + # the breadcrumbs will be start at that lane -- we won't see + # the sublane's parent or the library root. + sublane.root_for_patron_type = ["ya"] + assert_breadcrumbs([], sublane) + + assert_breadcrumbs([sublane, subsublane], subsubsublane) + + assert_breadcrumbs( + [sublane, subsublane, subsubsublane], subsubsublane, include_lane=True + ) + + # However, if an entrypoint is selected we will see a + # breadcrumb for it between the patron root lane and its + # child. + assert_breadcrumbs([sublane, ep, subsublane], subsubsublane, entrypoint=ep) + + assert_breadcrumbs( + [sublane, ep, subsublane, subsubsublane], + subsubsublane, + entrypoint=ep, + include_lane=True, + ) + + def test_add_breadcrumb_links(self, db: DatabaseTransactionFixture): + class MockFeed(OPDSAcquisitionFeed): + add_link_calls = [] + add_breadcrumbs_call = None + current_entrypoint = None + + def add_link(self, href, **kwargs): + kwargs["href"] = href + self.add_link_calls.append(kwargs) + + def add_breadcrumbs(self, lane, entrypoint): + self.add_breadcrumbs_call = (lane, entrypoint) + + def show_current_entrypoint(self, entrypoint): + self.current_entrypoint = entrypoint + + annotator = MockAnnotator + feed = MockFeed("title", "url", [], MockAnnotator()) + + lane = db.lane() + sublane = db.lane(parent=lane) + ep = AudiobooksEntryPoint + feed.add_breadcrumb_links(sublane, ep) + + # add_link_to_feed was called twice, to create the 'start' and + # 'up' links. + start, up = feed.add_link_calls + assert "start" == start["rel"] + assert annotator.top_level_title() == start["title"] + + assert "up" == up["rel"] + assert lane.display_name == up["title"] + + # The Lane and EntryPoint were passed into add_breadcrumbs. + assert (sublane, ep) == feed.add_breadcrumbs_call + + # The EntryPoint was passed into show_current_entrypoint. + assert ep == feed.current_entrypoint + + def test_show_current_entrypoint(self, db: DatabaseTransactionFixture): + """Calling OPDSAcquisitionFeed.show_current_entrypoint annotates + the top-level tag with information about the currently + selected entrypoint, if any. + """ + feed = OPDSAcquisitionFeed( + "title", + "url", + [], + CirculationManagerAnnotator(None), + ) + + # No entry point, no annotation. + feed.show_current_entrypoint(None) + assert feed._feed.entrypoint is None + + ep = AudiobooksEntryPoint + feed.show_current_entrypoint(ep) + assert ep.URI == feed._feed.entrypoint + + def test_facet_links_unrecognized_facets(self): + # OPDSAcquisitionFeed.facet_links does not produce links for any + # facet groups or facets not known to the current version of + # the system, because it doesn't know what the links should look + # like. + class MockAnnotator: + def facet_url(self, new_facets): + return "url: " + new_facets + + class MockFacets: + @property + def facet_groups(self): + """Yield a facet group+facet 4-tuple that passes the test we're + running (which will be turned into a link), and then a + bunch that don't (which will be ignored). + """ + + # Real facet group, real facet + yield ( + Facets.COLLECTION_FACET_GROUP_NAME, + Facets.COLLECTION_FULL, + "try the featured collection instead", + True, + ) + + # Real facet group, nonexistent facet + yield ( + Facets.COLLECTION_FACET_GROUP_NAME, + "no such facet", + "this facet does not exist", + True, + ) + + # Nonexistent facet group, real facet + yield ( + "no such group", + Facets.COLLECTION_FULL, + "this facet exists but it's in a nonexistent group", + True, + ) + + # Nonexistent facet group, nonexistent facet + yield ( + "no such group", + "no such facet", + "i just don't know", + True, + ) + + class MockFeed(OPDSAcquisitionFeed): + links = [] + + @classmethod + def facet_link(cls, url, facet_title, group_title, selected): + # Return the passed-in objects as is. + return (url, facet_title, group_title, selected) + + annotator = MockAnnotator() + facets = MockFacets() + + # The only 4-tuple yielded by facet_groups was passed on to us. + # The link was run through MockAnnotator.facet_url(), + # and the human-readable titles were found using lookups. + # + # The other three 4-tuples were ignored since we don't know + # how to generate human-readable titles for them. + [[url, facet, group, selected]] = MockFeed.facet_links(annotator, facets) + assert "url: try the featured collection instead" == url + assert Facets.FACET_DISPLAY_TITLES[Facets.COLLECTION_FULL] == facet + assert Facets.GROUP_DISPLAY_TITLES[Facets.COLLECTION_FACET_GROUP_NAME] == group + assert True == selected + + def test_active_loans_for_with_holds( + self, db: DatabaseTransactionFixture, patch_url_for: PatchedUrlFor + ): + patron = db.patron() + work = db.work(with_license_pool=True) + hold, _ = work.active_license_pool().on_hold_to(patron) + + feed = OPDSAcquisitionFeed.active_loans_for( + None, patron, LibraryAnnotator(None, None, db.default_library()) + ) + assert feed.annotator.active_holds_by_work == {work: hold} + + def test_single_entry_loans_feed_errors(self, db: DatabaseTransactionFixture): + with pytest.raises(ValueError) as raised: + # Mandatory loans item was missing + OPDSAcquisitionFeed.single_entry_loans_feed(None, None) # type: ignore[arg-type] + assert str(raised.value) == "Argument 'item' must be non-empty" + + with pytest.raises(ValueError) as raised: + # Mandatory loans item was incorrect + OPDSAcquisitionFeed.single_entry_loans_feed(None, object()) # type: ignore[arg-type] + assert "Argument 'item' must be an instance of" in str(raised.value) + + # A work and pool that has no edition, will not have an entry + work = db.work(with_open_access_download=True) + pool = work.active_license_pool() + work.presentation_edition = None + pool.presentation_edition = None + response = OPDSAcquisitionFeed.single_entry_loans_feed(MagicMock(), pool) + assert isinstance(response, OPDSEntryResponse) + assert response.status_code == 403 + + def test_single_entry_loans_feed_default_annotator( + self, db: DatabaseTransactionFixture + ): + patron = db.patron() + work = db.work(with_license_pool=True) + pool = work.active_license_pool() + assert pool is not None + loan, _ = pool.loan_to(patron) + + with patch.object(OPDSAcquisitionFeed, "single_entry") as mock: + mock.return_value = None + response = OPDSAcquisitionFeed.single_entry_loans_feed(None, loan) + + assert response == None + assert mock.call_count == 1 + _work, annotator = mock.call_args[0] + assert isinstance(annotator, LibraryLoanAndHoldAnnotator) + assert _work == work + assert annotator.library == db.default_library() + + def test_single_entry_with_edition(self, db: DatabaseTransactionFixture): + work = db.work(with_license_pool=True) + annotator = object() + + with patch.object(OPDSAcquisitionFeed, "_create_entry") as mock: + OPDSAcquisitionFeed.single_entry( + work.presentation_edition, annotator, even_if_no_license_pool=True # type: ignore[arg-type] + ) + + assert mock.call_count == 1 + _work, _pool, _edition, _identifier, _annotator = mock.call_args[0] + assert _work == work + assert _pool == None + assert _edition == work.presentation_edition + assert _identifier == work.presentation_edition.primary_identifier + assert _annotator == annotator + + +class TestEntrypointLinkInsertionFixture: + db: DatabaseTransactionFixture + mock: Any + no_eps: WorkList + entrypoints: List[MediumEntryPoint] + wl: WorkList + lane: Lane + annotator: Type[MockAnnotator] + old_add_entrypoint_links: Callable + + +@pytest.fixture() +def entrypoint_link_insertion_fixture( + db, +) -> Generator[TestEntrypointLinkInsertionFixture, None, None]: + data = TestEntrypointLinkInsertionFixture() + data.db = db + + # Mock for AcquisitionFeed.add_entrypoint_links + class Mock: + def add_entrypoint_links(self, *args): + self.called_with = args + + data.mock = Mock() + + # A WorkList with no EntryPoints -- should not call the mock method. + data.no_eps = WorkList() + data.no_eps.initialize(library=db.default_library(), display_name="no_eps") + + # A WorkList with two EntryPoints -- may call the mock method + # depending on circumstances. + data.entrypoints = [AudiobooksEntryPoint, EbooksEntryPoint] # type: ignore[list-item] + data.wl = WorkList() + # The WorkList must have at least one child, or we won't generate + # a real groups feed for it. + data.lane = db.lane() + data.wl.initialize( + library=db.default_library(), + display_name="wl", + entrypoints=data.entrypoints, + children=[data.lane], + ) + + def works(_db, **kwargs): + """Mock WorkList.works so we don't need any actual works + to run the test. + """ + return [] + + data.no_eps.works = works # type: ignore[method-assign, assignment] + data.wl.works = works # type: ignore[method-assign, assignment] + + data.annotator = MockAnnotator + data.old_add_entrypoint_links = OPDSAcquisitionFeed.add_entrypoint_links + OPDSAcquisitionFeed.add_entrypoint_links = data.mock.add_entrypoint_links # type: ignore[method-assign] + yield data + OPDSAcquisitionFeed.add_entrypoint_links = data.old_add_entrypoint_links # type: ignore[method-assign] + + +class TestEntrypointLinkInsertion: + """Verify that the three main types of OPDS feeds -- grouped, + paginated, and search results -- will all include links to the same + feed but through a different entry point. + """ + + def test_groups( + self, + entrypoint_link_insertion_fixture: TestEntrypointLinkInsertionFixture, + external_search_patch_fixture: ExternalSearchPatchFixture, + ): + data, db, session = ( + entrypoint_link_insertion_fixture, + entrypoint_link_insertion_fixture.db, + entrypoint_link_insertion_fixture.db.session, + ) + + # When AcquisitionFeed.groups() generates a grouped + # feed, it will link to different entry points into the feed, + # assuming the WorkList has different entry points. + def run(wl=None, facets=None): + """Call groups() and see what add_entrypoint_links + was called with. + """ + data.mock.called_with = None + search = MockExternalSearchIndex() + feed = OPDSAcquisitionFeed.groups( + session, + "title", + "url", + wl, + MockAnnotator(), + None, + facets, + search, + ) + return data.mock.called_with + + # This WorkList has no entry points, so the mock method is not + # even called. + assert None == run(data.no_eps) + + # A WorkList with entry points does cause the mock method + # to be called. + facets = FeaturedFacets( + minimum_featured_quality=db.default_library().settings.minimum_featured_quality, + entrypoint=EbooksEntryPoint, + ) + feed, make_link, entrypoints, selected = run(data.wl, facets) + + # add_entrypoint_links was passed both possible entry points + # and the selected entry point. + assert data.wl.entrypoints == entrypoints + assert selected == EbooksEntryPoint + + # The make_link function that was passed in calls + # TestAnnotator.groups_url() when passed an EntryPoint. + assert "http://groups/?entrypoint=Book" == make_link(EbooksEntryPoint) + + def test_page( + self, entrypoint_link_insertion_fixture: TestEntrypointLinkInsertionFixture + ): + data, db, session = ( + entrypoint_link_insertion_fixture, + entrypoint_link_insertion_fixture.db, + entrypoint_link_insertion_fixture.db.session, + ) + + # When AcquisitionFeed.page() generates the first page of a paginated + # list, it will link to different entry points into the list, + # assuming the WorkList has different entry points. + + def run(wl=None, facets=None, pagination=None): + """Call page() and see what add_entrypoint_links + was called with. + """ + data.mock.called_with = None + private = object() + OPDSAcquisitionFeed.page( + session, + "title", + "url", + wl, + data.annotator(), + facets, + pagination, + MockExternalSearchIndex(), + ) + + return data.mock.called_with + + # The WorkList has no entry points, so the mock method is not + # even called. + assert None == run(data.no_eps) + + # Let's give the WorkList two possible entry points, and choose one. + facets = Facets.default(db.default_library()).navigate( + entrypoint=EbooksEntryPoint + ) + feed, make_link, entrypoints, selected = run(data.wl, facets) + + # This time, add_entrypoint_links was called, and passed both + # possible entry points and the selected entry point. + assert data.wl.entrypoints == entrypoints + assert selected == EbooksEntryPoint + + # The make_link function that was passed in calls + # TestAnnotator.feed_url() when passed an EntryPoint. The + # Facets object's other facet groups are propagated in this URL. + first_page_url = "http://wl/?available=all&collection=full&collectionName=All&distributor=All&entrypoint=Book&order=author" + assert first_page_url == make_link(EbooksEntryPoint) + + # Pagination information is not propagated through entry point links + # -- you always start at the beginning of the list. + pagination = Pagination(offset=100) + feed, make_link, entrypoints, selected = run(data.wl, facets, pagination) + assert first_page_url == make_link(EbooksEntryPoint) + + def test_search( + self, entrypoint_link_insertion_fixture: TestEntrypointLinkInsertionFixture + ): + data, db, session = ( + entrypoint_link_insertion_fixture, + entrypoint_link_insertion_fixture.db, + entrypoint_link_insertion_fixture.db.session, + ) + + # When OPDSAcquisitionFeed.search() generates the first page of + # search results, it will link to related searches for different + # entry points, assuming the WorkList has different entry points. + def run(wl=None, facets=None, pagination=None): + """Call search() and see what add_entrypoint_links + was called with. + """ + data.mock.called_with = None + OPDSAcquisitionFeed.search( + session, + "title", + "url", + wl, + None, + None, + pagination=pagination, + facets=facets, + annotator=data.annotator(), + ) + return data.mock.called_with + + # Mock search() so it never tries to return anything. + def mock_search(self, *args, **kwargs): + return [] + + data.no_eps.search = mock_search # type: ignore[method-assign, assignment] + data.wl.search = mock_search # type: ignore[method-assign, assignment] + + # This WorkList has no entry points, so the mock method is not + # even called. + assert None == run(data.no_eps) + + # The mock method is called for a WorkList that does have + # entry points. + facets = SearchFacets().navigate(entrypoint=EbooksEntryPoint) + assert isinstance(facets, SearchFacets) + feed, make_link, entrypoints, selected = run(data.wl, facets) + + # Since the SearchFacets has more than one entry point, + # the EverythingEntryPoint is prepended to the list of possible + # entry points. + assert [ + EverythingEntryPoint, + AudiobooksEntryPoint, + EbooksEntryPoint, + ] == entrypoints + + # add_entrypoint_links was passed the three possible entry points + # and the selected entry point. + assert selected == EbooksEntryPoint + + # The make_link function that was passed in calls + # TestAnnotator.search_url() when passed an EntryPoint. + first_page_url = "http://wl/?available=all&collection=full&entrypoint=Book&order=relevance&search_type=default" + assert first_page_url == make_link(EbooksEntryPoint) + + # Pagination information is not propagated through entry point links + # -- you always start at the beginning of the list. + pagination = Pagination(offset=100) + feed, make_link, entrypoints, selected = run(data.wl, facets, pagination) + assert first_page_url == make_link(EbooksEntryPoint) + + +class TestLookupAcquisitionFeed: + @staticmethod + def _feed(session: Session, annotator=VerboseAnnotator, **kwargs): + """Helper method to create a LookupAcquisitionFeed.""" + return LookupAcquisitionFeed( + "Feed Title", + "http://whatever.io", + [], + annotator(), + **kwargs, + ) + + @staticmethod + def _entry( + session: Session, identifier, work, annotator=VerboseAnnotator, **kwargs + ): + """Helper method to create an entry.""" + feed = TestLookupAcquisitionFeed._feed(session, annotator, **kwargs) + entry = feed.single_entry((identifier, work), feed.annotator) + if isinstance(entry, OPDSMessage): + return feed, entry + return feed, entry + + def test_create_entry_uses_specified_identifier( + self, db: DatabaseTransactionFixture + ): + # Here's a Work with two LicensePools. + work = db.work(with_open_access_download=True) + original_pool = work.license_pools[0] + edition, new_pool = db.edition( + with_license_pool=True, with_open_access_download=True + ) + work.license_pools.append(new_pool) + + # We can generate two different OPDS entries for a single work + # depending on which identifier we look up. + ignore, e1 = self._entry(db.session, original_pool.identifier, work) + assert original_pool.identifier.urn == e1.computed.identifier + assert original_pool.presentation_edition.title == e1.computed.title.text + assert new_pool.identifier.urn != e1.computed.identifier + assert new_pool.presentation_edition.title != e1.computed.title.text + + # Different identifier and pool = different information + i = new_pool.identifier + ignore, e2 = self._entry(db.session, i, work) + assert new_pool.identifier.urn == e2.computed.identifier + assert new_pool.presentation_edition.title == e2.computed.title.text + assert original_pool.presentation_edition.title != e2.computed.title.text + assert original_pool.identifier.urn != e2.computed.identifier + + def test_error_on_mismatched_identifier(self, db: DatabaseTransactionFixture): + """We get an error if we try to make it look like an Identifier lookup + retrieved a Work that's not actually associated with that Identifier. + """ + work = db.work(with_open_access_download=True) + + # Here's an identifier not associated with any LicensePool or + # Work. + identifier = db.identifier() + + # It doesn't make sense to make an OPDS feed out of that + # Identifier and a totally random Work. + expect_error = 'I tried to generate an OPDS entry for the identifier "%s" using a Work not associated with that identifier.' + feed, entry = self._entry(db.session, identifier, work) + assert entry == OPDSMessage(identifier.urn, 500, expect_error % identifier.urn) + + # Even if the Identifier does have a Work, if the Works don't + # match, we get the same error. + edition, lp = db.edition(with_license_pool=True) + feed, entry = self._entry(db.session, lp.identifier, work) + assert entry == OPDSMessage( + lp.identifier.urn, 500, expect_error % lp.identifier.urn + ) + + def test_error_when_work_has_no_licensepool(self, db: DatabaseTransactionFixture): + """Under most circumstances, a Work must have at least one + LicensePool for a lookup to succeed. + """ + + # Here's a work with no LicensePools. + work = db.work(title="Hello, World!", with_license_pool=False) + identifier = work.presentation_edition.primary_identifier + feed, entry = self._entry(db.session, identifier, work) + # By default, a work is treated as 'not in the collection' if + # there is no LicensePool for it. + isinstance(entry, OPDSMessage) + assert 404 == entry.status_code + assert "Identifier not found in collection" == entry.message + + def test_unfilfullable_work(self, db: DatabaseTransactionFixture): + work = db.work(with_open_access_download=True) + [pool] = work.license_pools + feed, entry = self._entry( + db.session, pool.identifier, work, MockUnfulfillableAnnotator + ) + expect = OPDSAcquisitionFeed.error_message( + pool.identifier, + 403, + "I know about this work but can offer no way of fulfilling it.", + ) + assert expect == entry + + +class TestNavigationFeedFixture: + db: DatabaseTransactionFixture + fiction: Lane + fantasy: Lane + romance: Lane + contemporary_romance: Lane + + +@pytest.fixture() +def navigation_feed_fixture( + db, +) -> TestNavigationFeedFixture: + data = TestNavigationFeedFixture() + data.db = db + data.fiction = db.lane("Fiction") + data.fantasy = db.lane("Fantasy", parent=data.fiction) + data.romance = db.lane("Romance", parent=data.fiction) + data.contemporary_romance = db.lane("Contemporary Romance", parent=data.romance) + return data + + +class TestNavigationFeed: + def test_add_entry(self): + feed = NavigationFeed("title", "http://navigation", None, None) + feed.add_entry("http://example.com", "Example", "text/html") + [entry] = feed._feed.data_entries + assert "Example" == entry.title + [link] = entry.links + assert "http://example.com" == link.href + assert "text/html" == link.type + assert "subsection" == link.rel + + def test_navigation_with_sublanes( + self, navigation_feed_fixture: TestNavigationFeedFixture + ): + data, db, session = ( + navigation_feed_fixture, + navigation_feed_fixture.db, + navigation_feed_fixture.db.session, + ) + + private = object() + response = NavigationFeed.navigation( + session, + "Navigation", + "http://navigation", + data.fiction, + MockAnnotator(), + ) + + # The media type of this response is different than from the + # typical OPDSFeedResponse. + assert OPDSFeed.NAVIGATION_FEED_TYPE == response.as_response().content_type + + feed = response._feed + + assert "Navigation" == feed.metadata["title"].text + [self_link] = feed.links + assert "http://navigation" == self_link.href + assert "self" == self_link.rel + assert "http://navigation" == feed.metadata["id"].text + [fantasy, romance] = sorted(feed.data_entries, key=lambda x: x.title or "") + + assert data.fantasy.display_name == fantasy.title + assert "http://%s/" % data.fantasy.id == fantasy.id + [fantasy_link] = fantasy.links + assert "http://%s/" % data.fantasy.id == fantasy_link.href + assert "subsection" == fantasy_link.rel + assert OPDSFeed.ACQUISITION_FEED_TYPE == fantasy_link.type + + assert data.romance.display_name == romance.title + assert "http://navigation/%s" % data.romance.id == romance.id + [romance_link] = romance.links + assert "http://navigation/%s" % data.romance.id == romance_link.href + assert "subsection" == romance_link.rel + assert OPDSFeed.NAVIGATION_FEED_TYPE == romance_link.type + + def test_navigation_without_sublanes( + self, navigation_feed_fixture: TestNavigationFeedFixture + ): + data, db, session = ( + navigation_feed_fixture, + navigation_feed_fixture.db, + navigation_feed_fixture.db.session, + ) + + feed = NavigationFeed.navigation( + session, "Navigation", "http://navigation", data.fantasy, MockAnnotator() + ) + parsed = feed._feed + assert "Navigation" == parsed.metadata["title"].text + [self_link] = parsed.links + assert "http://navigation" == self_link.href + assert "self" == self_link.rel + assert "http://navigation" == parsed.metadata["id"].text + [fantasy] = parsed.data_entries + + assert "All " + data.fantasy.display_name == fantasy.title + assert "http://%s/" % data.fantasy.id == fantasy.id + [fantasy_link] = fantasy.links + assert "http://%s/" % data.fantasy.id == fantasy_link.href + assert "subsection" == fantasy_link.rel + assert OPDSFeed.ACQUISITION_FEED_TYPE == fantasy_link.type diff --git a/tests/api/feed/test_opds_base.py b/tests/api/feed/test_opds_base.py new file mode 100644 index 0000000000..ead68ec710 --- /dev/null +++ b/tests/api/feed/test_opds_base.py @@ -0,0 +1,57 @@ +from flask import Request + +from core.feed.opds import get_serializer +from core.feed.serializer.opds import OPDS1Serializer +from core.feed.serializer.opds2 import OPDS2Serializer + + +class TestBaseOPDSFeed: + def test_get_serializer(self): + # The q-value should take priority + request = Request.from_values( + headers=dict( + Accept="application/atom+xml;q=0.8,application/opds+json;q=0.9" + ) + ) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS2Serializer) + + # Multiple additional key-value pairs don't matter + request = Request.from_values( + headers=dict( + Accept="application/atom+xml;profile=opds-catalog;kind=acquisition;q=0.08, application/opds+json;q=0.9" + ) + ) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS2Serializer) + + request = Request.from_values( + headers=dict( + Accept="application/atom+xml;profile=opds-catalog;kind=acquisition" + ) + ) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS1Serializer) + + # The default q-value should be 1, but opds2 specificity is higher + request = Request.from_values( + headers=dict( + Accept="application/atom+xml;profile=feed,application/opds+json;q=0.9" + ) + ) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS2Serializer) + + # The default q-value should sort above 0.9 + request = Request.from_values( + headers=dict(Accept="application/opds+json;q=0.9,application/atom+xml") + ) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS1Serializer) + + # Same q-values respect order of definition in the code + request = Request.from_values( + headers=dict( + Accept="application/opds+json;q=0.9,application/atom+xml;q=0.9" + ) + ) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS1Serializer) + + # No valid accept mimetype should default to OPDS1.x + request = Request.from_values(headers=dict(Accept="text/html")) + assert isinstance(get_serializer(request.accept_mimetypes), OPDS1Serializer) diff --git a/tests/api/feed/test_opds_serializer.py b/tests/api/feed/test_opds_serializer.py new file mode 100644 index 0000000000..afe28b71c5 --- /dev/null +++ b/tests/api/feed/test_opds_serializer.py @@ -0,0 +1,232 @@ +import datetime + +import pytz +from lxml import etree + +from core.feed.serializer.opds import OPDS1Serializer +from core.feed.types import ( + Acquisition, + Author, + FeedEntryType, + IndirectAcquisition, + Link, + WorkEntryData, +) +from core.util.opds_writer import OPDSFeed, OPDSMessage + + +class TestOPDSSerializer: + def test__serialize_feed_entry(self): + grandchild = FeedEntryType.create(text="grandchild", attr="gcattr") + child = FeedEntryType.create(text="child", attr="chattr", grandchild=grandchild) + parent = FeedEntryType.create(text="parent", attr="pattr", child=child) + + serialized = OPDS1Serializer()._serialize_feed_entry("parent", parent) + + assert serialized.tag == "parent" + assert serialized.text == "parent" + assert serialized.get("attr") == "pattr" + children = list(serialized) + assert len(children) == 1 + assert children[0].tag == "child" + assert children[0].text == "child" + assert children[0].get("attr") == "chattr" + children = list(children[0]) + assert len(children) == 1 + assert children[0].tag == "grandchild" + assert children[0].text == "grandchild" + assert children[0].get("attr") == "gcattr" + + def test__serialize_author_tag(self): + author = Author( + name="Author", + sort_name="sort_name", + role="role", + link=Link(href="http://author", title="link title"), + viaf="viaf", + family_name="family name", + wikipedia_name="wiki name", + lc="lc", + ) + + element = OPDS1Serializer()._serialize_author_tag("author", author) + + assert element.tag == "author" + assert element.get(f"{{{OPDSFeed.OPF_NS}}}role") == author.role + + expected_child_tags = [ + (f"{{{OPDSFeed.ATOM_NS}}}name", author.name, None), + (f"{{{OPDSFeed.SIMPLIFIED_NS}}}sort_name", author.sort_name, None), + ( + f"{{{OPDSFeed.SIMPLIFIED_NS}}}wikipedia_name", + author.wikipedia_name, + None, + ), + ("sameas", author.viaf, None), + ("sameas", author.lc, None), + ("link", None, dict(href=author.link.href, title=author.link.title)), + ] + + child: etree._Element + for expect in expected_child_tags: + tag, text, attrs = expect + + # element.find is not working for "link" :| + for child in element: + if child.tag == tag: + break + else: + assert False, f"Did not find {expect}" + + # Remove the element so we don't find it again + element.remove(child) + + # Assert the data + assert child.text == text + if attrs: + assert dict(child.attrib) == attrs + + # No more children + assert list(element) == [] + + def test__serialize_acquistion_link(self): + link = Acquisition( + href="http://acquisition", + holds_total="0", + copies_total="1", + availability_status="available", + indirect_acquisitions=[IndirectAcquisition(type="indirect")], + ) + element = OPDS1Serializer()._serialize_acquistion_link(link) + assert element.tag == "link" + assert dict(element.attrib) == dict(href=link.href) + + for child in element: + if child.tag == f"{{{OPDSFeed.OPDS_NS}}}indirectAcquisition": + assert child.get("type") == "indirect" + elif child.tag == f"{{{OPDSFeed.OPDS_NS}}}holds": + assert child.get("total") == "0" + elif child.tag == f"{{{OPDSFeed.OPDS_NS}}}copies": + assert child.get("total") == "1" + elif child.tag == f"{{{OPDSFeed.OPDS_NS}}}availability": + assert child.get("status") == "available" + + def test_serialize_work_entry(self): + data = WorkEntryData( + additionalType="type", + identifier="identifier", + pwid="permanent-work-id", + summary=FeedEntryType(text="summary"), + language=FeedEntryType(text="language"), + publisher=FeedEntryType(text="publisher"), + issued=datetime.datetime(2020, 2, 2, tzinfo=pytz.UTC), + published=FeedEntryType(text="published"), + updated=FeedEntryType(text="updated"), + title=FeedEntryType(text="title"), + subtitle=FeedEntryType(text="subtitle"), + series=FeedEntryType.create( + name="series", + link=Link(href="http://series", title="series title", rel="series"), + ), + imprint=FeedEntryType(text="imprint"), + authors=[Author(name="author")], + contributors=[Author(name="contributor")], + categories=[ + FeedEntryType.create(scheme="scheme", term="term", label="label") + ], + ratings=[FeedEntryType(text="rating")], + ) + + element = OPDS1Serializer().serialize_work_entry(data) + + assert ( + element.get(f"{{{OPDSFeed.SCHEMA_NS}}}additionalType") + == data.additionalType + ) + + child = element.xpath(f"id") + assert len(child) == 1 + assert child[0].text == data.identifier + + child = element.findall(f"{{{OPDSFeed.SIMPLIFIED_NS}}}pwid") + assert len(child) == 1 + assert child[0].text == data.pwid + + child = element.xpath("summary") + assert len(child) == 1 + assert child[0].text == data.summary.text + + child = element.findall(f"{{{OPDSFeed.DCTERMS_NS}}}language") + assert len(child) == 1 + assert child[0].text == data.language.text + + child = element.findall(f"{{{OPDSFeed.DCTERMS_NS}}}publisher") + assert len(child) == 1 + assert child[0].text == data.publisher.text + + child = element.findall(f"{{{OPDSFeed.DCTERMS_NS}}}issued") + assert len(child) == 1 + assert child[0].text == data.issued.date().isoformat() + + child = element.findall(f"published") + assert len(child) == 1 + assert child[0].text == data.published.text + + child = element.findall(f"updated") + assert len(child) == 1 + assert child[0].text == data.updated.text + + child = element.findall(f"title") + assert len(child) == 1 + assert child[0].text == data.title.text + + child = element.findall(f"{{{OPDSFeed.SCHEMA_NS}}}alternativeHeadline") + assert len(child) == 1 + assert child[0].text == data.subtitle.text + + child = element.findall(f"{{{OPDSFeed.SCHEMA_NS}}}series") + assert len(child) == 1 + assert child[0].get("name") == getattr(data.series, "name") + link = list(child[0])[0] + assert link.tag == "link" + assert link.get("title") == "series title" + assert link.get("href") == "http://series" + + child = element.findall(f"{{{OPDSFeed.BIB_SCHEMA_NS}}}publisherImprint") + assert len(child) == 1 + assert child[0].text == data.imprint.text + + child = element.findall(f"author") + assert len(child) == 1 + name_tag = list(child[0])[0] + assert name_tag.tag == f"{{{OPDSFeed.ATOM_NS}}}name" + assert name_tag.text == "author" + + child = element.findall(f"contributor") + assert len(child) == 1 + name_tag = list(child[0])[0] + assert name_tag.tag == f"{{{OPDSFeed.ATOM_NS}}}name" + assert name_tag.text == "contributor" + + child = element.findall(f"category") + assert len(child) == 1 + assert child[0].get("scheme") == "scheme" + assert child[0].get("term") == "term" + assert child[0].get("label") == "label" + + child = element.findall(f"Rating") + assert len(child) == 1 + assert child[0].text == data.ratings[0].text + + def test_serialize_work_entry_empty(self): + # A no-data work entry + element = OPDS1Serializer().serialize_work_entry(WorkEntryData()) + # This will create an empty tag + assert element.tag == "entry" + assert list(element) == [] + + def test_serialize_opds_message(self): + message = OPDSMessage("URN", 200, "Description") + serializer = OPDS1Serializer() + result = serializer.serialize_opds_message(message) + assert serializer.to_string(result) == serializer.to_string(message.tag) diff --git a/tests/api/test_controller_cm.py b/tests/api/test_controller_cm.py index f97796b238..af0a0229d7 100644 --- a/tests/api/test_controller_cm.py +++ b/tests/api/test_controller_cm.py @@ -4,9 +4,12 @@ from api.config import Configuration from api.controller import CirculationManager from api.custom_index import CustomIndexView -from api.opds import CirculationManagerAnnotator, LibraryAnnotator from api.problem_details import * from core.external_search import MockExternalSearchIndex +from core.feed.annotator.circulation import ( + CirculationManagerAnnotator, + LibraryAnnotator, +) from core.lane import Facets, WorkList from core.model import Admin, CachedFeed, ConfigurationSetting, create from core.model.discovery_service_registration import DiscoveryServiceRegistration diff --git a/tests/api/test_controller_crawlfeed.py b/tests/api/test_controller_crawlfeed.py index 0750124a2a..4dee039352 100644 --- a/tests/api/test_controller_crawlfeed.py +++ b/tests/api/test_controller_crawlfeed.py @@ -1,6 +1,7 @@ import json from contextlib import contextmanager from typing import Any +from unittest.mock import MagicMock import feedparser from flask import url_for @@ -11,10 +12,10 @@ CrawlableFacets, DynamicLane, ) -from api.opds import CirculationManagerAnnotator from api.problem_details import NO_SUCH_COLLECTION, NO_SUCH_LIST from core.external_search import MockSearchResult, SortKeyPagination -from core.opds import AcquisitionFeed +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.circulation import CirculationManagerAnnotator from core.problem_details import INVALID_INPUT from core.util.flask_util import Response from core.util.problem_detail import ProblemDetail @@ -30,7 +31,7 @@ def mock_crawlable_feed(self, circulation_fixture: CirculationControllerFixture) controller = circulation_fixture.manager.opds_feeds original = controller._crawlable_feed - def mock(title, url, worklist, annotator=None, feed_class=AcquisitionFeed): + def mock(title, url, worklist, annotator=None, feed_class=OPDSAcquisitionFeed): self._crawlable_feed_called_with = dict( title=title, url=url, @@ -70,7 +71,7 @@ def test_crawlable_library_feed( assert expect_url == kwargs.pop("url") assert library.name == kwargs.pop("title") assert None == kwargs.pop("annotator") - assert AcquisitionFeed == kwargs.pop("feed_class") + assert OPDSAcquisitionFeed == kwargs.pop("feed_class") # A CrawlableCollectionBasedLane has been set up to show # everything in any of the requested library's collections. @@ -173,7 +174,7 @@ def test_crawlable_list_feed( assert expect_url == kwargs.pop("url") assert customlist.name == kwargs.pop("title") assert None == kwargs.pop("annotator") - assert AcquisitionFeed == kwargs.pop("feed_class") + assert OPDSAcquisitionFeed == kwargs.pop("feed_class") # A CrawlableCustomListBasedLane was created to fetch only # the works in the custom list. @@ -190,7 +191,9 @@ class MockFeed: @classmethod def page(cls, **kwargs): self.page_called_with = kwargs - return Response("An OPDS feed") + feed = MagicMock() + feed.as_response.return_value = Response("An OPDS feed") + return feed work = circulation_fixture.db.work(with_open_access_download=True) diff --git a/tests/api/test_controller_loan.py b/tests/api/test_controller_loan.py index a8b3664f6c..e0eb5032bf 100644 --- a/tests/api/test_controller_loan.py +++ b/tests/api/test_controller_loan.py @@ -885,7 +885,7 @@ def test_fulfill_without_single_item_feed(self, loan_fixture: LoanFixture): authenticated = controller.authenticated_patron_from_request() loan_fixture.pool.loan_to(authenticated) with patch( - "api.controller.LibraryLoanAndHoldAnnotator.single_item_feed" + "api.controller.OPDSAcquisitionFeed.single_entry_loans_feed" ) as feed, patch.object(circulation, "fulfill") as fulfill: # Complex setup # The fulfillmentInfo should not be have response type diff --git a/tests/api/test_controller_opdsfeed.py b/tests/api/test_controller_opdsfeed.py index 5074db8136..dd81e7b5c5 100644 --- a/tests/api/test_controller_opdsfeed.py +++ b/tests/api/test_controller_opdsfeed.py @@ -8,14 +8,16 @@ from api.controller import CirculationManager from api.lanes import HasSeriesFacets, JackpotFacets, JackpotWorkList -from api.opds import LibraryAnnotator from api.problem_details import REMOTE_INTEGRATION_FAILED from core.app_server import load_facets_from_request from core.entrypoint import AudiobooksEntryPoint, EverythingEntryPoint from core.external_search import SortKeyPagination -from core.lane import Facets, FeaturedFacets, Lane, Pagination, SearchFacets, WorkList +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.circulation import LibraryAnnotator +from core.feed.navigation import NavigationFeed +from core.lane import Facets, FeaturedFacets, Pagination, SearchFacets, WorkList from core.model import CachedFeed, Edition -from core.opds import AcquisitionFeed, NavigationFacets, NavigationFeed +from core.opds import NavigationFacets from core.util.flask_util import Response from tests.fixtures.api_controller import CirculationControllerFixture, WorkSpec from tests.fixtures.library import LibraryFixture @@ -107,9 +109,6 @@ def test_feed( # index. assert 200 == response.status_code - assert ( - "max-age=%d" % Lane.MAX_CACHE_AGE in response.headers["Cache-Control"] - ) feed = feedparser.parse(response.data) assert {x.title for x in circulation_fixture.works} == { x["title"] for x in feed["entries"] @@ -171,7 +170,9 @@ class Mock: @classmethod def page(cls, **kwargs): self.called_with = kwargs - return Response("An OPDS feed") + resp = MagicMock() + resp.as_response.return_value = Response("An OPDS feed") + return resp sort_key = ["sort", "pagination", "key"] with circulation_fixture.request_context_with_library( @@ -228,9 +229,6 @@ def page(cls, **kwargs): "search_engine" ) - # max age - assert 10 == kwargs.pop("max_age") - # No other arguments were passed into page(). assert {} == kwargs @@ -293,7 +291,9 @@ def groups(cls, **kwargs): # the grouped feed controller is activated. self.groups_called_with = kwargs self.page_called_with = None - return Response("A grouped feed") + resp = MagicMock() + resp.as_response.return_value = Response("A grouped feed") + return resp @classmethod def page(cls, **kwargs): @@ -301,7 +301,9 @@ def page(cls, **kwargs): # ends up being called instead. self.groups_called_with = None self.page_called_with = kwargs - return Response("A paginated feed") + resp = MagicMock() + resp.as_response.return_value = Response("A paginated feed") + return resp # Earlier we tested an authenticated request for a patron with an # external type. Now try an authenticated request for a patron with @@ -327,7 +329,6 @@ def page(cls, **kwargs): # The Response returned by Mock.groups() has been converted # into a Flask response. - assert 200 == response.status_code assert "A grouped feed" == response.get_data(as_text=True) # While we're in request context, generate the URL we @@ -504,7 +505,9 @@ class Mock: @classmethod def search(cls, **kwargs): self.called_with = kwargs - return "An OPDS feed" + resp = MagicMock() + resp.as_response.return_value = "An OPDS feed" + return resp with circulation_fixture.request_context_with_library( "/?q=t&size=99&after=22&media=Music" @@ -783,7 +786,7 @@ def test_qa_feed(self, circulation_fixture: CirculationControllerFixture): # For the most part, we're verifying that the expected values # are passed in to _qa_feed. - assert AcquisitionFeed.groups == kwargs.pop("feed_factory") # type: ignore + assert OPDSAcquisitionFeed.groups == kwargs.pop("feed_factory") # type: ignore assert JackpotFacets == kwargs.pop("facet_class") # type: ignore assert "qa_feed" == kwargs.pop("controller_name") # type: ignore assert "QA test feed" == kwargs.pop("feed_title") # type: ignore @@ -822,7 +825,7 @@ def test_qa_feed2(self, circulation_fixture: CirculationControllerFixture): # For the most part, we're verifying that the expected values # are passed in to _qa_feed. - assert AcquisitionFeed.groups == kwargs.pop("feed_factory") # type: ignore + assert OPDSAcquisitionFeed.groups == kwargs.pop("feed_factory") # type: ignore assert JackpotFacets == kwargs.pop("facet_class") # type: ignore assert "qa_feed" == kwargs.pop("controller_name") # type: ignore assert "QA test feed" == kwargs.pop("feed_title") # type: ignore @@ -865,7 +868,7 @@ def test_qa_series_feed(self, circulation_fixture: CirculationControllerFixture) # Note that the feed_method is different from the one in qa_feed. # We want to generate an ungrouped feed rather than a grouped one. - assert AcquisitionFeed.page == kwargs.pop("feed_factory") # type: ignore + assert OPDSAcquisitionFeed.page == kwargs.pop("feed_factory") # type: ignore assert HasSeriesFacets == kwargs.pop("facet_class") # type: ignore assert "qa_series_feed" == kwargs.pop("controller_name") # type: ignore assert "QA series test feed" == kwargs.pop("feed_title") # type: ignore diff --git a/tests/api/test_controller_work.py b/tests/api/test_controller_work.py index 0712ab298a..acc1a61f19 100644 --- a/tests/api/test_controller_work.py +++ b/tests/api/test_controller_work.py @@ -2,6 +2,7 @@ import json import urllib.parse from typing import Any, Dict +from unittest.mock import MagicMock import feedparser import flask @@ -18,15 +19,16 @@ SeriesLane, ) from api.novelist import MockNoveListAPI -from api.opds import LibraryAnnotator from api.problem_details import NO_SUCH_LANE, NOT_FOUND_ON_REMOTE from core.classifier import Classifier from core.entrypoint import AudiobooksEntryPoint from core.external_search import SortKeyPagination, mock_search_index +from core.feed.acquisition import OPDSAcquisitionFeed +from core.feed.annotator.circulation import LibraryAnnotator +from core.feed.types import WorkEntry from core.lane import Facets, FeaturedFacets from core.metadata_layer import ContributorData, Metadata from core.model import ( - CachedFeed, DataSource, Edition, Identifier, @@ -37,7 +39,6 @@ tuple_to_numericrange, ) from core.model.work import Work -from core.opds import AcquisitionFeed from core.problem_details import INVALID_INPUT from core.util.datetime_helpers import utc_now from core.util.flask_util import Response @@ -138,11 +139,6 @@ def test_contributor(self, work_fixture: WorkFixture): ] assert 10 == len(facet_links) - # The feed was cached. - cached = work_fixture.db.session.query(CachedFeed).one() - assert CachedFeed.CONTRIBUTOR_TYPE == cached.type - assert "John Bull-eng,spa-Children,Young+Adult" == cached.unique_key - # At this point we don't want to generate real feeds anymore. # We can't do a real end-to-end test without setting up a real # search index, which is obnoxiously slow. @@ -164,7 +160,9 @@ class Mock: @classmethod def page(cls, **kwargs): self.called_with = kwargs - return Response("An OPDS feed") + resp = MagicMock() + resp.as_response.return_value = Response("An OPDS feed") + return resp # Test a basic request with custom faceting, pagination, and a # language and audience restriction. This will exercise nearly @@ -291,12 +289,12 @@ def test_permalink(self, work_fixture: WorkFixture): work_fixture.identifier.type, work_fixture.identifier.identifier ) annotator = LibraryAnnotator(None, None, work_fixture.db.default_library()) - expect = AcquisitionFeed.single_entry( - work_fixture.db.session, work_fixture.english_1, annotator - ).data + feed = OPDSAcquisitionFeed.single_entry(work_fixture.english_1, annotator) + assert isinstance(feed, WorkEntry) + expect = OPDSAcquisitionFeed.entry_as_response(feed) assert 200 == response.status_code - assert expect == response.get_data() + assert expect.data == response.get_data() assert OPDSFeed.ENTRY_TYPE == response.headers["Content-Type"] def test_permalink_does_not_return_fulfillment_links_for_authenticated_patrons_without_loans( @@ -334,9 +332,9 @@ def test_permalink_does_not_return_fulfillment_links_for_authenticated_patrons_w work_fixture.db.default_library(), active_loans_by_work=active_loans_by_work, ) - expect = AcquisitionFeed.single_entry( - work_fixture.db.session, work, annotator - ).data + feed = OPDSAcquisitionFeed.single_entry(work, annotator) + assert isinstance(feed, WorkEntry) + expect = OPDSAcquisitionFeed.entry_as_response(feed).data response = work_fixture.manager.work_controller.permalink( identifier_type, identifier @@ -382,9 +380,9 @@ def test_permalink_returns_fulfillment_links_for_authenticated_patrons_with_loan work_fixture.db.default_library(), active_loans_by_work=active_loans_by_work, ) - expect = AcquisitionFeed.single_entry( - work_fixture.db.session, work, annotator - ).data + feed = OPDSAcquisitionFeed.single_entry(work, annotator) + assert isinstance(feed, WorkEntry) + expect = OPDSAcquisitionFeed.entry_as_response(feed).data response = work_fixture.manager.work_controller.permalink( identifier_type, identifier @@ -475,9 +473,9 @@ def test_permalink_returns_fulfillment_links_for_authenticated_patrons_with_fulf work_fixture.db.default_library(), active_loans_by_work=active_loans_by_work, ) - expect = AcquisitionFeed.single_entry( - work_fixture.db.session, work, annotator - ).data + feed = OPDSAcquisitionFeed.single_entry(work, annotator) + assert isinstance(feed, WorkEntry) + expect = OPDSAcquisitionFeed.entry_as_response(feed).data response = work_fixture.manager.work_controller.permalink( identifier_type, identifier @@ -561,7 +559,9 @@ class Mock: @classmethod def page(cls, **kwargs): cls.called_with = kwargs - return Response("A bunch of titles") + resp = MagicMock() + resp.as_response.return_value = Response("A bunch of titles") + return resp kwargs["feed_class"] = Mock with work_fixture.request_context_with_library( @@ -754,7 +754,9 @@ class Mock: @classmethod def groups(cls, **kwargs): cls.called_with = kwargs - return Response("An OPDS feed") + resp = MagicMock() + resp.as_response.return_value = Response("An OPDS feed") + return resp mock_api.setup_method(metadata) with work_fixture.request_context_with_library("/?entrypoint=Audio"): @@ -818,7 +820,7 @@ def groups(cls, **kwargs): **url_kwargs, ) assert kwargs.pop("url") == expect_url - + assert kwargs.pop("pagination") == None # That's it! assert {} == kwargs @@ -910,14 +912,6 @@ def test_series(self, work_fixture: WorkFixture): assert "Sort by" == series_position["opds:facetgroup"] assert "true" == series_position["opds:activefacet"] - # The feed was cached. - cached = work_fixture.db.session.query(CachedFeed).one() - assert CachedFeed.SERIES_TYPE == cached.type - assert ( - "Like As If Whatever Mysteries-eng,spa-Children,Young+Adult" - == cached.unique_key - ) - # At this point we don't want to generate real feeds anymore. # We can't do a real end-to-end test without setting up a real # search index, which is obnoxiously slow. @@ -938,7 +932,9 @@ class Mock: @classmethod def page(cls, **kwargs): self.called_with = kwargs - return Response("An OPDS feed") + resp = MagicMock() + resp.as_response.return_value = Response("An OPDS feed") + return resp # Test a basic request with custom faceting, pagination, and a # language and audience restriction. This will exercise nearly diff --git a/tests/api/test_scripts.py b/tests/api/test_scripts.py index db2e62a92c..50846c5697 100644 --- a/tests/api/test_scripts.py +++ b/tests/api/test_scripts.py @@ -337,7 +337,9 @@ class MockAcquisitionFeed: @classmethod def page(cls, **kwargs): cls.called_with = kwargs - return "here's your feed" + resp = MagicMock() + resp.as_response.return_value = "here's your feed" + return resp # Test our ability to generate a single feed. script = CacheFacetListsPerLane(db.session, testing=True, cmd_args=[]) @@ -355,7 +357,6 @@ def page(cls, **kwargs): assert db.session == args["_db"] # type: ignore assert lane == args["worklist"] # type: ignore assert lane.display_name == args["title"] # type: ignore - assert 0 == args["max_age"] # type: ignore # The Pagination object was passed into # MockAcquisitionFeed.page, and it was also used to make the @@ -415,7 +416,9 @@ class MockAcquisitionFeed: @classmethod def groups(cls, **kwargs): cls.called_with = kwargs - return "here's your feed" + resp = MagicMock() + resp.as_response.return_value = "here's your feed" + return resp # Test our ability to generate a single feed. script = CacheOPDSGroupFeedPerLane(db.session, testing=True, cmd_args=[]) @@ -433,7 +436,6 @@ def groups(cls, **kwargs): assert db.session == args["_db"] # type: ignore assert lane == args["worklist"] # type: ignore assert lane.display_name == args["title"] # type: ignore - assert 0 == args["max_age"] # type: ignore assert pagination == None # The Facets object was passed into @@ -503,6 +505,8 @@ def test_facets( (no_entry_point,) = script.facets(lane) assert None == no_entry_point.entrypoint + # We no longer cache the feeds + @pytest.mark.skip def test_do_run(self, lane_script_fixture: LaneScriptFixture): db = lane_script_fixture.db diff --git a/tests/core/test_app_server.py b/tests/core/test_app_server.py index 9a07373f68..da73365833 100644 --- a/tests/core/test_app_server.py +++ b/tests/core/test_app_server.py @@ -22,10 +22,10 @@ ) from core.config import Configuration from core.entrypoint import AudiobooksEntryPoint, EbooksEntryPoint +from core.feed.annotator.base import Annotator from core.lane import Facets, Pagination, SearchFacets, WorkList from core.log import LogConfiguration from core.model import ConfigurationSetting, Identifier -from core.opds import MockAnnotator from core.problem_details import INVALID_INPUT, INVALID_URN from core.util.opds_writer import OPDSFeed, OPDSMessage from tests.fixtures.database import DatabaseTransactionFixture @@ -265,7 +265,7 @@ def test_work_lookup( work = data.transaction.work(with_license_pool=True) identifier = work.license_pools[0].identifier - annotator = MockAnnotator() + annotator = Annotator() # NOTE: We run this test twice to verify that the controller # doesn't keep any state between requests. At one point there # was a bug which would have caused a book to show up twice on @@ -311,7 +311,7 @@ def test_permalink(self, urn_lookup_controller_fixture: URNLookupControllerFixtu work = data.transaction.work(with_license_pool=True) work.license_pools[0].open_access = False identifier = work.license_pools[0].identifier - annotator = MockAnnotator() + annotator = Annotator() with data.app.test_request_context("/?urn=%s" % identifier.urn): response = data.controller.permalink(identifier.urn, annotator)