Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

♻️ [#2060] Replace get_paginated_results with pagination_helper #995

Merged
merged 3 commits into from
Feb 12, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -666,7 +666,7 @@ def test_categories_based_on_cases_for_eherkenning_user_with_vestigingsnummer(
self.assertEqual(context["categories"][3], self.category7)

@patch(
"open_inwoner.openzaak.cases.get_paginated_results",
"zgw_consumers.service.pagination_helper",
side_effect=RequestException,
)
def test_categories_fail_to_fetch_cases(self, m):
14 changes: 6 additions & 8 deletions src/open_inwoner/haalcentraal/api.py
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@

from open_inwoner.haalcentraal.api_models import BRPData
from open_inwoner.haalcentraal.models import HaalCentraalConfig
from open_inwoner.utils.api import ClientError, JSONParserClient
from open_inwoner.utils.api import ClientError, get_json_response

logger = logging.getLogger(__name__)

@@ -25,9 +25,7 @@ def __init__(self):
if not self.config.service:
logger.warning("no service defined for Haal Centraal")
else:
self.client = build_client(
self.config.service, client_factory=JSONParserClient
)
self.client = build_client(self.config.service)
self._is_ready = True

@abc.abstractmethod
@@ -74,7 +72,7 @@ def fetch_data(self, user_bsn: str) -> Optional[dict]:
headers["x-doelbinding"] = self.config.api_doelbinding

try:
data = self.client.get(
response = self.client.get(
url=url,
headers=headers,
params={
@@ -87,7 +85,7 @@ def fetch_data(self, user_bsn: str) -> Optional[dict]:
},
verify=False,
)
return data
return get_json_response(response)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return None
@@ -121,7 +119,7 @@ class BRP_2_1(BRPAPI):
def fetch_data(self, user_bsn: str) -> Optional[dict]:
url = urljoin(self.client.base_url, "personen")
try:
data = self.client.post(
response = self.client.post(
url=url,
data={
"fields": [
@@ -145,7 +143,7 @@ def fetch_data(self, user_bsn: str) -> Optional[dict]:
headers={"Accept": "application/json"},
verify=False,
)
return data
return get_json_response(response)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return None
234 changes: 227 additions & 7 deletions src/open_inwoner/openklant/clients.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,246 @@
import logging
from typing import Optional
from typing import List, Optional

from ape_pie.client import APIClient
from requests.exceptions import RequestException
from zgw_consumers.api_models.base import factory
from zgw_consumers.client import build_client as _build_client
from zgw_consumers.utils import pagination_helper

from open_inwoner.utils.api import JSONParserClient
from open_inwoner.openzaak.cases import fetch_case_by_url_no_cache
from open_inwoner.utils.api import ClientError, get_json_response

from .api_models import (
ContactMoment,
ContactMomentCreateData,
Klant,
KlantContactMoment,
KlantContactRol,
KlantCreateData,
ObjectContactMoment,
)
from .models import OpenKlantConfig

logger = logging.getLogger(__name__)


class KlantenClient(APIClient):
def create_klant(self, data: KlantCreateData) -> Optional[Klant]:
try:
response = self.post("klanten", json=data)
data = get_json_response(response)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return

klant = factory(Klant, data)

return klant

def retrieve_klant(
self, user_bsn: Optional[str] = None, user_kvk_or_rsin: Optional[str] = None
) -> Optional[Klant]:
if not user_bsn and not user_kvk_or_rsin:
return

# this is technically a search operation and could return multiple records
if user_bsn:
klanten = self.retrieve_klanten_for_bsn(user_bsn)
elif user_kvk_or_rsin:
klanten = self.retrieve_klanten_for_kvk_or_rsin(user_kvk_or_rsin)

if klanten:
# let's use the first one
return klanten[0]
else:
return

def retrieve_klanten_for_bsn(self, user_bsn: str) -> List[Klant]:
try:
response = self.get(
"klanten",
params={"subjectNatuurlijkPersoon__inpBsn": user_bsn},
)
data = get_json_response(response)
all_data = list(pagination_helper(self, data))
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return []

klanten = factory(Klant, all_data)

return klanten

def retrieve_klanten_for_kvk_or_rsin(
self, user_kvk_or_rsin: str, *, vestigingsnummer=None
) -> List[Klant]:
params = {"subjectNietNatuurlijkPersoon__innNnpId": user_kvk_or_rsin}

if vestigingsnummer:
params = {
"subjectVestiging__vestigingsNummer": vestigingsnummer,
}

try:
response = self.get(
"klanten",
params=params,
)
data = get_json_response(response)
all_data = list(pagination_helper(self, data))
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return []

klanten = factory(Klant, all_data)

return klanten

def partial_update_klant(self, klant: Klant, update_data) -> Optional[Klant]:
try:
response = self.patch(url=klant.url, json=update_data)
data = get_json_response(response)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return

klant = factory(Klant, data)

return klant


class ContactmomentenClient(APIClient):
def create_contactmoment(
self,
data: ContactMomentCreateData,
*,
klant: Optional[Klant] = None,
rol: Optional[str] = KlantContactRol.BELANGHEBBENDE,
) -> Optional[ContactMoment]:
try:
response = self.post("contactmomenten", json=data)
data = get_json_response(response)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return

contactmoment = factory(ContactMoment, data)

if klant:
# relate contact to klant though a klantcontactmoment
try:
response = self.post(
"klantcontactmomenten",
json={
"klant": klant.url,
"contactmoment": contactmoment.url,
"rol": rol,
},
)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return

return contactmoment

def retrieve_contactmoment(self, url) -> Optional[ContactMoment]:
try:
response = self.get(url)
data = get_json_response(response)
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return

contact_moment = factory(ContactMoment, data)

return contact_moment

def retrieve_objectcontactmomenten_for_contactmoment(
self, contactmoment: ContactMoment
) -> List[ObjectContactMoment]:
try:
response = self.get(
"objectcontactmomenten", params={"contactmoment": contactmoment.url}
)
data = get_json_response(response)
all_data = list(pagination_helper(self, data))
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return []

object_contact_momenten = factory(ObjectContactMoment, all_data)

# resolve linked resources
object_mapping = {}
for ocm in object_contact_momenten:
assert ocm.contactmoment == contactmoment.url
ocm.contactmoment = contactmoment
if ocm.object_type == "zaak":
object_url = ocm.object
# Avoid fetching the same object, if multiple relations wit the same object exist
if ocm.object in object_mapping:
ocm.object = object_mapping[object_url]
else:
ocm.object = fetch_case_by_url_no_cache(ocm.object)
object_mapping[object_url] = ocm.object

return object_contact_momenten

def retrieve_klantcontactmomenten_for_klant(
self, klant: Klant
) -> List[KlantContactMoment]:
try:
response = self.get(
"klantcontactmomenten",
params={"klant": klant.url},
)
data = get_json_response(response)
all_data = list(pagination_helper(self, data))
except (RequestException, ClientError) as e:
logger.exception("exception while making request", exc_info=e)
return []

klanten_contact_moments = factory(KlantContactMoment, all_data)

# resolve linked resources
for kcm in klanten_contact_moments:
assert kcm.klant == klant.url
kcm.klant = klant
kcm.contactmoment = self.retrieve_contactmoment(kcm.contactmoment)

return klanten_contact_moments

def retrieve_objectcontactmomenten_for_object_type(
self, contactmoment: ContactMoment, object_type: str
) -> List[ObjectContactMoment]:

moments = self.retrieve_objectcontactmomenten_for_contactmoment(contactmoment)

# eSuite doesn't implement a `object_type` query parameter
ret = [moment for moment in moments if moment.object_type == object_type]

return ret
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Since we're not using the list (?), you could:
  • yield from (moment for moment in moments if...)
  • (in the following function): return next(ocms, None)
  1. This seems to be the only use of the function which potentially returns multiple objects, but then we're only retrieving and returning the first (in retrieve_objectcontactmoment). Is this to allow for possible extensions where we actually use multiple objectcontactmomenten from this? Otherwise this and the following function could be collapsed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a API so best not to mix lists and generator return values (because you don't know how it is used).

(also: the example would be return (moment for moment in moments if...) (adding yield from just adds another generator around a generator))

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could collapse it into one function retrieve_objectcontactmoment, since retrieve_objectcontactmomenten_for_object_type is indeed only used by that function currently. Though we might need this in the future, so that's why I added it initially


def retrieve_objectcontactmoment(
self, contactmoment: ContactMoment, object_type: str
) -> Optional[ObjectContactMoment]:
ocms = self.retrieve_objectcontactmomenten_for_object_type(
contactmoment, object_type
)
if ocms:
return ocms[0]


def build_client(type_) -> Optional[APIClient]:
config = OpenKlantConfig.get_solo()
services = {
"klanten",
"contactmomenten",
services_to_client_mapping = {
"klanten": KlantenClient,
"contactmomenten": ContactmomentenClient,
}
if type_ in services:
if client_class := services_to_client_mapping.get(type_):
service = getattr(config, f"{type_}_service")
if service:
client = _build_client(service, client_factory=JSONParserClient)
client = _build_client(service, client_factory=client_class)
return client

logger.warning(f"no service defined for {type_}")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logger.warning("no service defined for {type}", type=type_) (better to avoid f-string interpolation with logging)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(better to avoid f-string interpolation with logging)

@pi-sigma Nooooo. Where does this information/advice come from?

.format() is much more dangerous because it'll actually raise runtime errors if you miss a replacement placeholder, while f-strings would catch it at parse time.

Copy link

@Viicos Viicos Feb 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logger.warning("no service defined for %s", type_) is actually the correct form (see https://docs.python.org/3/howto/logging.html#optimization)

("no service defined for {type}", type=type_ is afaik not supported by the logging module)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That links to the optimization section. Here is the bit about variables: https://docs.python.org/3/howto/logging.html#logging-variable-data, which doesn't mention a correct form but says something about support.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought logger.warning("no service defined for {type}", type=type_) was equivalent to logger.warning("no service defined for %s", type_); perhaps that's not the case.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought logger.warning("no service defined for {type}", type=type_) was equivalent to logger.warning("no service defined for %s", type_); perhaps that's not the case.

log methods only allow pos. args (except for exc_info and similar). I also think that apart from performance (which can be meaningless is most cases), it allows easier grouping of log messages in tools like Sentry

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Bartvaderkin This not not (merely) about performance. The main motivation is to facilitate log aggregation in Sentry. If you do logger.warning("no service defined for %s", type_), the logger can group together logs with different instances under the same label. Both f-strings and .format should be avoided. Like I said, I didn't think of logger.warning("no service defined for {type}", type=type_) as equivalent to .format() logging.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we doing log aggregation in tools like Sentry in this project? I don't think the existing logs are setup for that so that would be introducing something new.

Copy link

@Viicos Viicos Feb 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it's true that with deferred string interpolation, it would only error at runtime if there's a mismatch in the number of arguments. I think linting rules exist to warn on this issue though (logging-too-many-args from Pylint)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we doing log aggregation in tools like Sentry in this project? I don't think the existing logs are setup for that so that would be introducing something new.

We do have Sentry set up for test/acceptance/prod and if logger.warning(f"no service defined for {type_}") gets triggered for two different type_s, it will create two separate issues in Sentry, whereas logger.warning(f"no service defined for %s", type_) will group them together

Loading