From c41ab1a57fcea3003066bb6421f535428cd107dd Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 13 Apr 2023 16:28:26 -0400 Subject: [PATCH 01/15] feat: Nation Builder connector - get_people gets all people in NB account --- parsons/__init__.py | 3 +- parsons/nation_builder/__init__.py | 3 + parsons/nation_builder/nation_builder.py | 71 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 parsons/nation_builder/__init__.py create mode 100644 parsons/nation_builder/nation_builder.py diff --git a/parsons/__init__.py b/parsons/__init__.py index bd30b0fff9..eff0c761cf 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -1,7 +1,7 @@ # Provide shortcuts to importing Parsons submodules and set up logging import importlib -import os import logging +import os from parsons.etl.table import Table @@ -63,6 +63,7 @@ ("parsons.hustle.hustle", "Hustle"), ("parsons.mailchimp.mailchimp", "Mailchimp"), ("parsons.mobilize_america.ma", "MobilizeAmerica"), + ("parsons.nation_builder.nation_builder", "NationBuilder"), ("parsons.newmode.newmode", "Newmode"), ("parsons.ngpvan.van", "VAN"), ("parsons.notifications.gmail", "Gmail"), diff --git a/parsons/nation_builder/__init__.py b/parsons/nation_builder/__init__.py new file mode 100644 index 0000000000..3520433033 --- /dev/null +++ b/parsons/nation_builder/__init__.py @@ -0,0 +1,3 @@ +from parsons.nation_builder.nation_builder import NationBuilder + +__all__ = ["NationBuilder"] diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py new file mode 100644 index 0000000000..25d944542b --- /dev/null +++ b/parsons/nation_builder/nation_builder.py @@ -0,0 +1,71 @@ +import logging +import time +from typing import Optional +from urllib.parse import parse_qs, urlparse + +from parsons import Table +from parsons.utilities import check_env +from parsons.utilities.api_connector import APIConnector + +logger = logging.getLogger(__name__) + + +class NationBuilder: + """ + Instantiate the NationBuilder class + + `Args:` + slug: str + The Nation Builder slug Not required if ``NB_SLUG`` env variable set. The slug is the + nation slug of the nation from which your application is requesting approval to retrieve + data via the NationBuilder API. For example, your application's user could provide this + slug via a text field in your application. + access_token: str + The Nation Builder access_token Not required if ``NB_ACCESS_TOKEN`` env variable set. + """ + + def __init__( + self, slug: Optional[str] = None, access_token: Optional[str] = None + ) -> None: + self.slug = check_env.check("NB_SLUG", slug) + self.access_token = check_env.check("NB_ACCESS_TOKEN", access_token) + + self.uri = f"https://{self.slug}.nationbuilder.com/api/v1" + + self.client = APIConnector( + self.uri, headers={"authorization": f"Bearer {self.access_token}"} + ) + + def get_people(self): + """ + `Returns:` + A Table of all people stored in Nation Builder. + """ + data = [] + original_url = "people?limit=100" + + url = f"{original_url}" + + while True: + try: + logging.debug("sending request %s" % url) + response = self.client.get_request(url) + + res = response["results"] + logging.debug("response got %s records" % len(res)) + + data.extend(res) + + if response["next"]: + next_params = parse_qs(urlparse(response["next"]).query) + nonce = next_params["__nonce"][0] + token = next_params["__token"][0] + url = f"{original_url}&__nonce={nonce}&__token={token}" + else: + break + except Exception as error: + logging.error("error requesting data from Nation Builder: %s" % error) + wait_time = 30 + time.sleep(wait_time) + + return Table(data) From d484dc965b0175994131c9e836c80ef53f506fb3 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 13 Apr 2023 17:32:29 -0400 Subject: [PATCH 02/15] refactor for testability --- parsons/nation_builder/nation_builder.py | 63 +++++++++++++++++++----- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index 25d944542b..042764666f 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -1,6 +1,6 @@ import logging import time -from typing import Optional +from typing import Dict, Optional, Tuple from urllib.parse import parse_qs, urlparse from parsons import Table @@ -27,22 +27,65 @@ class NationBuilder: def __init__( self, slug: Optional[str] = None, access_token: Optional[str] = None ) -> None: - self.slug = check_env.check("NB_SLUG", slug) - self.access_token = check_env.check("NB_ACCESS_TOKEN", access_token) - - self.uri = f"https://{self.slug}.nationbuilder.com/api/v1" + slug = check_env.check("NB_SLUG", slug) + token = check_env.check("NB_ACCESS_TOKEN", access_token) self.client = APIConnector( - self.uri, headers={"authorization": f"Bearer {self.access_token}"} + NationBuilder.get_uri(slug), headers=NationBuilder.get_auth_headers(token) ) + @classmethod + def get_uri(cls, slug: Optional[str]) -> str: + if slug is None: + raise ValueError("slug can't None") + + if not isinstance(slug, str): + raise ValueError("slug must be an str") + + if len(slug.strip()) == 0: + raise ValueError("slug can't be an empty str") + + return f"https://{slug}.nationbuilder.com/api/v1" + + @classmethod + def get_auth_headers(cls, access_token: Optional[str]) -> Dict[str, str]: + if access_token is None: + raise ValueError("access_token can't None") + + if not isinstance(access_token, str): + raise ValueError("access_token must be an str") + + if len(access_token.strip()) == 0: + raise ValueError("access_token can't be an empty str") + + return {"authorization": f"Bearer {access_token}"} + + @classmethod + def parse_next_params(cls, next_value: str) -> Tuple[str, str]: + next_params = parse_qs(urlparse(next_value).query) + + if "__nonce" not in next_params: + raise ValueError("__nonce param not found") + + if "__token" not in next_params: + raise ValueError("__token param not found") + + nonce = next_params["__nonce"][0] + token = next_params["__token"][0] + + return nonce, token + + @classmethod + def make_next_url(cls, original_url: str, nonce: str, token: str) -> str: + return f"{original_url}?limit=100&__nonce={nonce}&__token={token}" + def get_people(self): """ `Returns:` A Table of all people stored in Nation Builder. """ data = [] - original_url = "people?limit=100" + original_url = "people" url = f"{original_url}" @@ -57,10 +100,8 @@ def get_people(self): data.extend(res) if response["next"]: - next_params = parse_qs(urlparse(response["next"]).query) - nonce = next_params["__nonce"][0] - token = next_params["__token"][0] - url = f"{original_url}&__nonce={nonce}&__token={token}" + nonce, token = NationBuilder.parse_next_params(response["next"]) + url = NationBuilder.make_next_url(original_url, nonce, token) else: break except Exception as error: From 52a661455dbf6821022222c4c635199c454a7c59 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 13 Apr 2023 17:32:47 -0400 Subject: [PATCH 03/15] add tests for class methods --- test/test_nation_builder/__init__.py | 0 .../test_nation_builder.py | 55 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 test/test_nation_builder/__init__.py create mode 100644 test/test_nation_builder/test_nation_builder.py diff --git a/test/test_nation_builder/__init__.py b/test/test_nation_builder/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/test_nation_builder/test_nation_builder.py b/test/test_nation_builder/test_nation_builder.py new file mode 100644 index 0000000000..d41081fa3d --- /dev/null +++ b/test/test_nation_builder/test_nation_builder.py @@ -0,0 +1,55 @@ +import unittest + +import requests_mock + +from parsons import NationBuilder as NB + + +class TestNationBuilder(unittest.TestCase): + def test_client(self): + nb = NB("test-slug", "test-token") + self.assertEqual(nb.client.uri, "https://test-slug.nationbuilder.com/api/v1/") + self.assertEqual(nb.client.headers, {"authorization": "Bearer test-token"}) + + def test_get_uri_success(self): + self.assertEqual(NB.get_uri("foo"), "https://foo.nationbuilder.com/api/v1") + self.assertEqual(NB.get_uri("bar"), "https://bar.nationbuilder.com/api/v1") + + def test_get_uri_errors(self): + values = ["", " ", None, 1337, {}, []] + + for v in values: + with self.assertRaises(ValueError): + NB.get_uri(v) + + def test_get_auth_headers_success(self): + self.assertEqual(NB.get_auth_headers("foo"), {"authorization": "Bearer foo"}) + self.assertEqual(NB.get_auth_headers("bar"), {"authorization": "Bearer bar"}) + + def test_get_auth_headers_errors(self): + values = ["", " ", None, 1337, {}, []] + + for v in values: + with self.assertRaises(ValueError): + NB.get_auth_headers(v) + + def test_parse_next_params_success(self): + n, t = NB.parse_next_params("/a/b/c?__nonce=foo&__token=bar") + self.assertEqual(n, "foo") + self.assertEqual(t, "bar") + + def test_get_next_params_errors(self): + with self.assertRaises(ValueError): + NB.parse_next_params("/a/b/c?baz=1") + + with self.assertRaises(ValueError): + NB.parse_next_params("/a/b/c?__nonce=1") + + with self.assertRaises(ValueError): + NB.parse_next_params("/a/b/c?__token=1") + + def test_make_next_url(self): + self.assertEqual( + NB.make_next_url("example.com", "bar", "baz"), + "example.com?limit=100&__nonce=bar&__token=baz", + ) From 5272f483e2f31a96c64d27f9626975af062a7973 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Mon, 5 Jun 2023 11:22:25 -0400 Subject: [PATCH 04/15] add a waiting log message --- parsons/nation_builder/nation_builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index 042764666f..73680625f1 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -106,7 +106,9 @@ def get_people(self): break except Exception as error: logging.error("error requesting data from Nation Builder: %s" % error) + wait_time = 30 + logging.info("waiting %d seconds before retrying" % wait_time) time.sleep(wait_time) return Table(data) From c7092ff163e81dd030eff7a18a00928433aad875 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 6 Jun 2023 15:46:19 -0400 Subject: [PATCH 05/15] feat: update_person by id for nation builder --- parsons/nation_builder/nation_builder.py | 37 +++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index 73680625f1..835cdff51b 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -1,6 +1,7 @@ +import json import logging import time -from typing import Dict, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, cast from urllib.parse import parse_qs, urlparse from parsons import Table @@ -30,9 +31,10 @@ def __init__( slug = check_env.check("NB_SLUG", slug) token = check_env.check("NB_ACCESS_TOKEN", access_token) - self.client = APIConnector( - NationBuilder.get_uri(slug), headers=NationBuilder.get_auth_headers(token) - ) + headers = {"Content-Type": "application/json", "Accept": "application/json"} + headers.update(NationBuilder.get_auth_headers(token)) + + self.client = APIConnector(NationBuilder.get_uri(slug), headers=headers) @classmethod def get_uri(cls, slug: Optional[str]) -> str: @@ -112,3 +114,30 @@ def get_people(self): time.sleep(wait_time) return Table(data) + + def update_person(self, person_id: str, data: Dict[str, Any]) -> dict[str, Any]: + """ + `Args:` + person_id: str + Nation Builder person id. + data: dict + Nation builder person object. + For example {"email": "user@example.com", "tags": ["foo", "bar"]} + Docs: https://nationbuilder.com/people_api + `Returns:` + A person object with the updated data. + """ + if person_id is None: + raise ValueError("person_id can't None") + + if not isinstance(person_id, str): + raise ValueError("person_id must be an str") + + if len(person_id.strip()) == 0: + raise ValueError("person_id can't be an empty str") + + url = f"people/{person_id}" + response = self.client.put_request(url, data=json.dumps({"person": data})) + response = cast(dict[str, Any], response) + + return response From 1e545658d10ebdf0ce3e10c1cedc78b3bb1001b8 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 6 Jun 2023 16:04:03 -0400 Subject: [PATCH 06/15] feat: add upsert_person to Nation Builder --- parsons/nation_builder/nation_builder.py | 53 +++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index 835cdff51b..6576644900 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -115,8 +115,10 @@ def get_people(self): return Table(data) - def update_person(self, person_id: str, data: Dict[str, Any]) -> dict[str, Any]: + def update_person(self, person_id: str, person: Dict[str, Any]) -> dict[str, Any]: """ + Updates a person with the provided data. + `Args:` person_id: str Nation Builder person id. @@ -137,7 +139,54 @@ def update_person(self, person_id: str, data: Dict[str, Any]) -> dict[str, Any]: raise ValueError("person_id can't be an empty str") url = f"people/{person_id}" - response = self.client.put_request(url, data=json.dumps({"person": data})) + response = self.client.put_request(url, data=json.dumps({"person": person})) response = cast(dict[str, Any], response) return response + + def upsert_person( + self, person: Dict[str, Any] + ) -> Tuple[bool, dict[str, Any] | None]: + """ + Updates a matched person or creates a new one if the person doesn't exist. + + This method attempts to match the input person resource to a person already in the + nation. If a match is found, the matched person is updated and a 200 status code is + returned. If a match is not found, a new person is created and a 201 status code is + returned. Matches are found by including one of the following IDs in the request: + + - civicrm_id + - county_file_id + - dw_id + - external_id + - email + - facebook_username + - ngp_id + - salesforce_id + - twitter_login + - van_id + + `Args:` + data: dict + Nation builder person object. + For example {"email": "user@example.com", "tags": ["foo", "bar"]} + Docs: https://nationbuilder.com/people_api + `Returns:` + A tuple of `created` and `person` object with the updated data. If the request fails + the method will return a tuple of `False` and `None`. + """ + + url = "people/push" + response = self.client.request(url, "PUT", data=json.dumps({"person": person})) + + self.client.validate_response(response) + + if response.status_code == 200: + if self.client.json_check(response): + return (False, response.json()) + + if response.status_code == 201: + if self.client.json_check(response): + return (True, response.json()) + + return (False, None) From cad0680c3a7926286c6101928682e2f0c9987165 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 6 Jun 2023 16:05:54 -0400 Subject: [PATCH 07/15] fix: update method docs --- parsons/nation_builder/nation_builder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index 6576644900..ca30576a7b 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -117,7 +117,8 @@ def get_people(self): def update_person(self, person_id: str, person: Dict[str, Any]) -> dict[str, Any]: """ - Updates a person with the provided data. + This method updates a person with the provided id to have the provided data. It returns a + full representation of the updated person. `Args:` person_id: str @@ -151,9 +152,8 @@ def upsert_person( Updates a matched person or creates a new one if the person doesn't exist. This method attempts to match the input person resource to a person already in the - nation. If a match is found, the matched person is updated and a 200 status code is - returned. If a match is not found, a new person is created and a 201 status code is - returned. Matches are found by including one of the following IDs in the request: + nation. If a match is found, the matched person is updated. If a match is not found, a new + person is created. Matches are found by including one of the following IDs in the request: - civicrm_id - county_file_id From 5097980e108ecb36bb4702abdd5e9f53a114e500 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 6 Jun 2023 16:18:19 -0400 Subject: [PATCH 08/15] fix: add return type hint to get_people --- parsons/nation_builder/nation_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index ca30576a7b..654d5efd06 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -81,7 +81,7 @@ def parse_next_params(cls, next_value: str) -> Tuple[str, str]: def make_next_url(cls, original_url: str, nonce: str, token: str) -> str: return f"{original_url}?limit=100&__nonce={nonce}&__token={token}" - def get_people(self): + def get_people(self) -> Table: """ `Returns:` A Table of all people stored in Nation Builder. From bf78076d8aa7f2747c63c823c32fc8553cbd29d3 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 6 Jun 2023 16:28:46 -0400 Subject: [PATCH 09/15] text: add missing header fields to test --- test/test_nation_builder/test_nation_builder.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/test_nation_builder/test_nation_builder.py b/test/test_nation_builder/test_nation_builder.py index d41081fa3d..fc71a44697 100644 --- a/test/test_nation_builder/test_nation_builder.py +++ b/test/test_nation_builder/test_nation_builder.py @@ -9,7 +9,14 @@ class TestNationBuilder(unittest.TestCase): def test_client(self): nb = NB("test-slug", "test-token") self.assertEqual(nb.client.uri, "https://test-slug.nationbuilder.com/api/v1/") - self.assertEqual(nb.client.headers, {"authorization": "Bearer test-token"}) + self.assertEqual( + nb.client.headers, + { + "authorization": "Bearer test-token", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) def test_get_uri_success(self): self.assertEqual(NB.get_uri("foo"), "https://foo.nationbuilder.com/api/v1") From 2ff44add938265ae4d560e5dc6abbb48df4bb871 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 6 Jun 2023 16:33:33 -0400 Subject: [PATCH 10/15] test: remove unused import in test --- test/test_nation_builder/test_nation_builder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_nation_builder/test_nation_builder.py b/test/test_nation_builder/test_nation_builder.py index fc71a44697..034e53c059 100644 --- a/test/test_nation_builder/test_nation_builder.py +++ b/test/test_nation_builder/test_nation_builder.py @@ -1,7 +1,5 @@ import unittest -import requests_mock - from parsons import NationBuilder as NB From 756ee7f2b6b24b57cf95aaafdf7cd8056360e883 Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Tue, 20 Jun 2023 14:57:42 -0400 Subject: [PATCH 11/15] docs: add NationBuilder class docs --- docs/index.rst | 1 + docs/nation_builder.rst | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 docs/nation_builder.rst diff --git a/docs/index.rst b/docs/index.rst index 0ce117ddac..2183d89469 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -205,6 +205,7 @@ Indices and tables hustle mailchimp mobilize_america + nation_builder newmode ngpvan p2a diff --git a/docs/nation_builder.rst b/docs/nation_builder.rst new file mode 100644 index 0000000000..649f52dace --- /dev/null +++ b/docs/nation_builder.rst @@ -0,0 +1,52 @@ +NationBuilder +============== + +******** +Overview +******** + +The NationBuilder class allows you to interact with the NationBuilder API. Users of this Parsons integration can download a full list of people, update and upsert people. + +.. note:: + Authentication + In order to use this class you need your nation slug and access token. To get your access token login to your nation and navigate to ``Settings > Developer > API Token`` and create a new token. You can get more info in the `NationBuilder API docs `_. + +========== +Quickstart +========== + +To instantiate the NationBuilder class, you can either store your ``NB_SLUG`` and ``NB_ACCESS_TOKEN`` keys as environment +variables or pass them in as arguments: + +.. code-block:: python + + from parsons import NationBuilder + + # First approach: Use API key environment variables + + # In bash, set your environment variables like so: + # export NB_SLUG='my-nation-slug' + # export NB_ACCESS_TOKEN='MY_ACCESS_TOKEN' + nb = NationBuilder() + + # Second approach: Pass API keys as arguments + nb = NationBuilder(slug='my-nation-slug', access_token='MY_ACCESS_TOKEN') + +You can then make a request to get all people and save its data to a Parsons table using the method, ``get_people()``: + +.. code-block:: python + + # Create Parsons table with people data from API + parsons_table = nb.get_people() + + # Save people as CSV + parsons_table.to_csv('people.csv') + +The above example shows how to create a Parsons table with all people registered in your NationBuilder nation. + +*** +API +*** + +.. autoclass :: parsons.NationBuilder + :inherited-members: From b8f485f81531f3e9e8dfa39fa7119c4522f4670a Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 22 Jun 2023 16:34:33 -0400 Subject: [PATCH 12/15] feat: add more attr validation --- parsons/nation_builder/nation_builder.py | 35 ++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py index 654d5efd06..6a6ff6a920 100644 --- a/parsons/nation_builder/nation_builder.py +++ b/parsons/nation_builder/nation_builder.py @@ -96,12 +96,16 @@ def get_people(self) -> Table: logging.debug("sending request %s" % url) response = self.client.get_request(url) - res = response["results"] + res = response.get("results", None) + + if res is None: + break + logging.debug("response got %s records" % len(res)) data.extend(res) - if response["next"]: + if response.get("next", None): nonce, token = NationBuilder.parse_next_params(response["next"]) url = NationBuilder.make_next_url(original_url, nonce, token) else: @@ -134,11 +138,14 @@ def update_person(self, person_id: str, person: Dict[str, Any]) -> dict[str, Any raise ValueError("person_id can't None") if not isinstance(person_id, str): - raise ValueError("person_id must be an str") + raise ValueError("person_id must be a str") if len(person_id.strip()) == 0: raise ValueError("person_id can't be an empty str") + if not isinstance(person, dict): + raise ValueError("person must be a dict") + url = f"people/{person_id}" response = self.client.put_request(url, data=json.dumps({"person": person})) response = cast(dict[str, Any], response) @@ -176,6 +183,28 @@ def upsert_person( the method will return a tuple of `False` and `None`. """ + _required_keys = [ + "civicrm_id", + "county_file_id", + "dw_id", + "external_id", + "email", + "facebook_username", + "ngp_id", + "salesforce_id", + "twitter_login", + "van_id", + ] + + if not isinstance(person, dict): + raise ValueError("person must be a dict") + + has_required_key = any(x in person for x in _required_keys) + + if not has_required_key: + _keys = ", ".join(_required_keys) + raise ValueError(f"person dict must contain at least one key of {_keys}") + url = "people/push" response = self.client.request(url, "PUT", data=json.dumps({"person": person})) From c1b0a0125b39b16da7cbc0fdda11de5c5a3140ec Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 22 Jun 2023 16:34:50 -0400 Subject: [PATCH 13/15] test: request testing using fixtures --- test/test_nation_builder/fixtures.py | 350 ++++++++++++++++++ .../test_nation_builder.py | 115 ++++++ 2 files changed, 465 insertions(+) create mode 100644 test/test_nation_builder/fixtures.py diff --git a/test/test_nation_builder/fixtures.py b/test/test_nation_builder/fixtures.py new file mode 100644 index 0000000000..90b1f9db84 --- /dev/null +++ b/test/test_nation_builder/fixtures.py @@ -0,0 +1,350 @@ +GET_PEOPLE_RESPONSE = { + "results": [ + { + "birthdate": None, + "city_district": None, + "civicrm_id": None, + "county_district": None, + "county_file_id": None, + "created_at": "2023-06-22T11:41:56-07:00", + "datatrust_id": None, + "do_not_call": False, + "do_not_contact": False, + "dw_id": None, + "email": "foo@example.com", + "email_opt_in": True, + "employer": None, + "external_id": None, + "federal_district": None, + "fire_district": None, + "first_name": "Foo", + "has_facebook": False, + "id": 4, + "is_twitter_follower": False, + "is_volunteer": False, + "judicial_district": None, + "labour_region": None, + "last_name": "Bar", + "linkedin_id": None, + "mobile": None, + "mobile_opt_in": None, + "middle_name": "", + "nbec_guid": None, + "ngp_id": None, + "note": None, + "occupation": None, + "party": None, + "pf_strat_id": None, + "phone": None, + "precinct_id": None, + "primary_address": None, + "profile_image_url_ssl": "https://assets.nationbuilder.com/assets/notifier/profile-avatar.png", + "recruiter_id": None, + "rnc_id": None, + "rnc_regid": None, + "salesforce_id": None, + "school_district": None, + "school_sub_district": None, + "sex": None, + "signup_type": 0, + "state_file_id": None, + "state_lower_district": None, + "state_upper_district": None, + "support_level": None, + "supranational_district": None, + "tags": ["zoot", "boot"], + "twitter_id": None, + "twitter_name": None, + "updated_at": "2023-06-22T11:41:56-07:00", + "van_id": None, + "village_district": None, + "ward": None, + "work_phone_number": None, + }, + { + "birthdate": None, + "city_district": None, + "civicrm_id": None, + "county_district": None, + "county_file_id": None, + "created_at": "2023-06-22T08:21:00-07:00", + "datatrust_id": None, + "do_not_call": False, + "do_not_contact": False, + "dw_id": None, + "email": "bar@example.com", + "email_opt_in": True, + "employer": None, + "external_id": None, + "federal_district": None, + "fire_district": None, + "first_name": "Zoo", + "has_facebook": False, + "id": 2, + "is_twitter_follower": False, + "is_volunteer": False, + "judicial_district": None, + "labour_region": None, + "last_name": "Baz", + "linkedin_id": None, + "mobile": None, + "mobile_opt_in": True, + "middle_name": "", + "nbec_guid": None, + "ngp_id": None, + "note": None, + "occupation": None, + "party": None, + "pf_strat_id": None, + "phone": None, + "precinct_id": None, + "primary_address": None, + "profile_image_url_ssl": "https://assets.nationbuilder.com/assets/notifier/profile-avatar.png", + "recruiter_id": None, + "rnc_id": None, + "rnc_regid": None, + "salesforce_id": None, + "school_district": None, + "school_sub_district": None, + "sex": None, + "signup_type": 0, + "state_file_id": None, + "state_lower_district": None, + "state_upper_district": None, + "support_level": None, + "supranational_district": None, + "tags": ["zoo", "bar"], + "twitter_id": None, + "twitter_name": None, + "updated_at": "2023-06-22T11:43:03-07:00", + "van_id": None, + "village_district": None, + "ward": None, + "work_phone_number": None, + }, + ], + "next": None, + "prev": None, +} + +PERSON_RESPONSE = { + "person": { + "birthdate": None, + "city_district": None, + "civicrm_id": None, + "county_district": None, + "county_file_id": None, + "created_at": "2023-06-22T08:21:00-07:00", + "datatrust_id": None, + "do_not_call": False, + "do_not_contact": False, + "dw_id": None, + "email": "foo@example.com", + "email_opt_in": True, + "employer": None, + "external_id": None, + "federal_district": None, + "fire_district": None, + "first_name": "Foo", + "has_facebook": False, + "id": 1, + "is_twitter_follower": False, + "is_volunteer": False, + "judicial_district": None, + "labour_region": None, + "last_name": "Bar", + "linkedin_id": None, + "mobile": None, + "mobile_opt_in": True, + "middle_name": "", + "nbec_guid": None, + "ngp_id": None, + "note": None, + "occupation": None, + "party": None, + "pf_strat_id": None, + "phone": None, + "precinct_id": None, + "primary_address": None, + "profile_image_url_ssl": "https://assets.nationbuilder.com/assets/notifier/profile-avatar.png", + "recruiter_id": None, + "rnc_id": None, + "rnc_regid": None, + "salesforce_id": None, + "school_district": None, + "school_sub_district": None, + "sex": None, + "signup_type": 0, + "state_file_id": None, + "state_lower_district": None, + "state_upper_district": None, + "support_level": None, + "supranational_district": None, + "tags": [], + "twitter_id": None, + "twitter_name": None, + "updated_at": "2023-06-22T08:21:00-07:00", + "van_id": None, + "village_district": None, + "ward": None, + "work_phone_number": None, + "active_customer_expires_at": None, + "active_customer_started_at": None, + "author": None, + "author_id": None, + "auto_import_id": None, + "availability": None, + "ballots": [], + "banned_at": None, + "billing_address": None, + "bio": None, + "call_status_id": None, + "call_status_name": None, + "capital_amount_in_cents": 500, + "children_count": 0, + "church": None, + "city_sub_district": None, + "closed_invoices_amount_in_cents": None, + "closed_invoices_count": None, + "contact_status_id": None, + "contact_status_name": None, + "could_vote_status": False, + "demo": None, + "donations_amount_in_cents": 0, + "donations_amount_this_cycle_in_cents": 0, + "donations_count": 0, + "donations_count_this_cycle": 0, + "donations_pledged_amount_in_cents": 0, + "donations_raised_amount_in_cents": 0, + "donations_raised_amount_this_cycle_in_cents": 0, + "donations_raised_count": 0, + "donations_raised_count_this_cycle": 0, + "donations_to_raise_amount_in_cents": 0, + "email1": "foo@example.com", + "email1_is_bad": False, + "email2": None, + "email2_is_bad": False, + "email3": None, + "email3_is_bad": False, + "email4": None, + "email4_is_bad": False, + "emails": [ + { + "email_address": "foo@example.com", + "email_number": 1, + "is_bad": False, + "is_primary": True, + } + ], + "ethnicity": None, + "facebook_address": None, + "facebook_profile_url": None, + "facebook_updated_at": None, + "facebook_username": None, + "fax_number": None, + "federal_donotcall": False, + "first_donated_at": None, + "first_fundraised_at": None, + "first_invoice_at": None, + "first_prospect_at": None, + "first_recruited_at": None, + "first_supporter_at": "2023-06-22T08:21:00-07:00", + "first_volunteer_at": None, + "full_name": "Foo Bar", + "home_address": None, + "import_id": None, + "inferred_party": None, + "inferred_support_level": None, + "invoice_payments_amount_in_cents": None, + "invoice_payments_referred_amount_in_cents": None, + "invoices_amount_in_cents": None, + "invoices_count": None, + "is_absentee_voter": None, + "is_active_voter": None, + "is_deceased": False, + "is_donor": False, + "is_dropped_from_file": None, + "is_early_voter": None, + "is_fundraiser": False, + "is_ignore_donation_limits": False, + "is_leaderboardable": True, + "is_mobile_bad": False, + "is_permanent_absentee_voter": None, + "is_possible_duplicate": False, + "is_profile_private": False, + "is_profile_searchable": True, + "is_prospect": False, + "is_supporter": True, + "is_survey_question_private": False, + "language": None, + "last_call_id": None, + "last_contacted_at": None, + "last_contacted_by": None, + "last_donated_at": None, + "last_fundraised_at": None, + "last_invoice_at": None, + "last_rule_violation_at": None, + "legal_name": None, + "locale": None, + "mailing_address": None, + "marital_status": None, + "media_market_name": None, + "meetup_id": None, + "meetup_address": None, + "mobile_normalized": None, + "nbec_precinct_code": None, + "nbec_precinct": None, + "note_updated_at": None, + "outstanding_invoices_amount_in_cents": None, + "outstanding_invoices_count": None, + "overdue_invoices_count": None, + "page_slug": None, + "parent": None, + "parent_id": None, + "party_member": None, + "phone_normalized": None, + "phone_time": None, + "precinct_code": None, + "precinct_name": None, + "prefix": None, + "previous_party": None, + "primary_email_id": 1, + "priority_level": None, + "priority_level_changed_at": None, + "profile_content": None, + "profile_content_html": None, + "profile_headline": None, + "received_capital_amount_in_cents": 500, + "recruiter": None, + "recruits_count": 0, + "registered_address": None, + "registered_at": None, + "religion": None, + "rule_violations_count": 0, + "signup_sources": [], + "spent_capital_amount_in_cents": 0, + "submitted_address": None, + "subnations": [], + "suffix": None, + "support_level_changed_at": None, + "support_probability_score": None, + "township": None, + "turnout_probability_score": None, + "twitter_address": None, + "twitter_description": None, + "twitter_followers_count": None, + "twitter_friends_count": None, + "twitter_location": None, + "twitter_login": None, + "twitter_updated_at": None, + "twitter_website": None, + "unsubscribed_at": None, + "user_submitted_address": None, + "username": None, + "voter_updated_at": None, + "warnings_count": 0, + "website": None, + "work_address": None, + }, + "precinct": None, +} diff --git a/test/test_nation_builder/test_nation_builder.py b/test/test_nation_builder/test_nation_builder.py index 034e53c059..f458632a36 100644 --- a/test/test_nation_builder/test_nation_builder.py +++ b/test/test_nation_builder/test_nation_builder.py @@ -1,7 +1,11 @@ import unittest +import requests_mock + from parsons import NationBuilder as NB +from .fixtures import GET_PEOPLE_RESPONSE, PERSON_RESPONSE + class TestNationBuilder(unittest.TestCase): def test_client(self): @@ -58,3 +62,114 @@ def test_make_next_url(self): NB.make_next_url("example.com", "bar", "baz"), "example.com?limit=100&__nonce=bar&__token=baz", ) + + @requests_mock.Mocker() + def test_get_people_handle_empty_response(self, m): + nb = NB("test-slug", "test-token") + m.get("https://test-slug.nationbuilder.com/api/v1/people", json={"results": []}) + table = nb.get_people() + self.assertEqual(table.num_rows, 0) + + @requests_mock.Mocker() + def test_get_people(self, m): + nb = NB("test-slug", "test-token") + m.get( + "https://test-slug.nationbuilder.com/api/v1/people", + json=GET_PEOPLE_RESPONSE, + ) + table = nb.get_people() + + self.assertEqual(table.num_rows, 2) + self.assertEqual(len(table.columns), 59) + + self.assertEqual(table[0]["first_name"], "Foo") + self.assertEqual(table[0]["last_name"], "Bar") + self.assertEqual(table[0]["email"], "foo@example.com") + + @requests_mock.Mocker() + def test_get_people_with_next(self, m): + """Make two requests and get the same data twice. This will exercise the while loop.""" + nb = NB("test-slug", "test-token") + + GET_PEOPLE_RESPONSE_WITH_NEXT = GET_PEOPLE_RESPONSE.copy() + GET_PEOPLE_RESPONSE_WITH_NEXT[ + "next" + ] = "https://test-slug.nationbuilder.com/api/v1/people?limit=100&__nonce=bar&__token=baz" + + m.get( + "https://test-slug.nationbuilder.com/api/v1/people", + json=GET_PEOPLE_RESPONSE_WITH_NEXT, + ) + + m.get( + "https://test-slug.nationbuilder.com/api/v1/people?limit=100&__nonce=bar&__token=baz", + json=GET_PEOPLE_RESPONSE, + ) + + table = nb.get_people() + + self.assertEqual(table.num_rows, 4) + self.assertEqual(len(table.columns), 59) + + self.assertEqual(table[1]["first_name"], "Zoo") + self.assertEqual(table[1]["last_name"], "Baz") + self.assertEqual(table[1]["email"], "bar@example.com") + + def test_update_person_raises_with_bad_params(self): + nb = NB("test-slug", "test-token") + + with self.assertRaises(ValueError): + nb.update_person(None, {}) + + with self.assertRaises(ValueError): + nb.update_person(1, {}) + + with self.assertRaises(ValueError): + nb.update_person(" ", {}) + + with self.assertRaises(ValueError): + nb.update_person("1", None) + + with self.assertRaises(ValueError): + nb.update_person("1", "bad value") + + @requests_mock.Mocker() + def test_update_person(self, m): + """Requests the correct URL, returns the correct data and doesn't raise exceptions.""" + nb = NB("test-slug", "test-token") + + m.put( + "https://test-slug.nationbuilder.com/api/v1/people/1", + json=PERSON_RESPONSE, + ) + + response = nb.update_person("1", {"tags": ["zoot", "boot"]}) + person = response["person"] + + self.assertEqual(person["id"], 1) + self.assertEqual(person["first_name"], "Foo") + self.assertEqual(person["last_name"], "Bar") + self.assertEqual(person["email"], "foo@example.com") + + @requests_mock.Mocker() + def test_upsert_person(self, m): + """Requests the correct URL, returns the correct data and doesn't raise exceptions.""" + nb = NB("test-slug", "test-token") + + m.put( + "https://test-slug.nationbuilder.com/api/v1/people/push", + json=PERSON_RESPONSE, + ) + + with self.assertRaises(ValueError): + nb.upsert_person({"tags": ["zoot", "boot"]}) + + created, response = nb.upsert_person({"email": "foo@example.com"}) + self.assertFalse(created) + + person = response["person"] + + self.assertEqual(person["id"], 1) + self.assertEqual(person["first_name"], "Foo") + self.assertEqual(person["last_name"], "Bar") + self.assertEqual(person["email"], "foo@example.com") From 9763566e16cb3554e0c8c440e1e758c8db18b28a Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 22 Jun 2023 16:43:31 -0400 Subject: [PATCH 14/15] linter: fix long lines in fixtures --- test/test_nation_builder/fixtures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_nation_builder/fixtures.py b/test/test_nation_builder/fixtures.py index 90b1f9db84..fb5a0eaee6 100644 --- a/test/test_nation_builder/fixtures.py +++ b/test/test_nation_builder/fixtures.py @@ -38,7 +38,7 @@ "phone": None, "precinct_id": None, "primary_address": None, - "profile_image_url_ssl": "https://assets.nationbuilder.com/assets/notifier/profile-avatar.png", + "profile_image_url_ssl": "https://example.com/assets/notifier/profile-avatar.png", "recruiter_id": None, "rnc_id": None, "rnc_regid": None, @@ -99,7 +99,7 @@ "phone": None, "precinct_id": None, "primary_address": None, - "profile_image_url_ssl": "https://assets.nationbuilder.com/assets/notifier/profile-avatar.png", + "profile_image_url_ssl": "https://example.com/assets/notifier/profile-avatar.png", "recruiter_id": None, "rnc_id": None, "rnc_regid": None, @@ -166,7 +166,7 @@ "phone": None, "precinct_id": None, "primary_address": None, - "profile_image_url_ssl": "https://assets.nationbuilder.com/assets/notifier/profile-avatar.png", + "profile_image_url_ssl": "https://example.com/assets/notifier/profile-avatar.png", "recruiter_id": None, "rnc_id": None, "rnc_regid": None, From 98f65145550cf935ff890a75026c6bf79bfa2ffa Mon Sep 17 00:00:00 2001 From: Giovanni Collazo Date: Thu, 22 Jun 2023 16:46:45 -0400 Subject: [PATCH 15/15] test: extract upsert_person params test --- test/test_nation_builder/test_nation_builder.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_nation_builder/test_nation_builder.py b/test/test_nation_builder/test_nation_builder.py index f458632a36..83c52ffe6f 100644 --- a/test/test_nation_builder/test_nation_builder.py +++ b/test/test_nation_builder/test_nation_builder.py @@ -151,6 +151,12 @@ def test_update_person(self, m): self.assertEqual(person["last_name"], "Bar") self.assertEqual(person["email"], "foo@example.com") + def test_upsert_person_raises_with_bad_params(self): + nb = NB("test-slug", "test-token") + + with self.assertRaises(ValueError): + nb.upsert_person({"tags": ["zoot", "boot"]}) + @requests_mock.Mocker() def test_upsert_person(self, m): """Requests the correct URL, returns the correct data and doesn't raise exceptions.""" @@ -161,9 +167,6 @@ def test_upsert_person(self, m): json=PERSON_RESPONSE, ) - with self.assertRaises(ValueError): - nb.upsert_person({"tags": ["zoot", "boot"]}) - created, response = nb.upsert_person({"email": "foo@example.com"}) self.assertFalse(created)