Merge pull request #538 from biocore/csymons_melissa_dupe_check_fix

Address Verification Adjustments

microsetta_private_api/api/_interested_user.py

@@ -105,18 +105,12 @@ def get_interested_user_address_update(interested_user_id, email):
                 else:
                     # We've determined it's a valid user and their address
                     # needs to be fixed, so we return a subset of their info.
-                    # Certain paths into the database allow null address_2
-                    # which causes problems, so we change it to "".
-                    if interested_user.address_2 is None:
-                        interested_user.address_2 = ""
-
                     # We also need to grab the reason(s) that Melissa couldn't
                     # verify the address.
                     melissa_repo = MelissaRepo(t)
                     melissa_result = melissa_repo.check_duplicate(
                         interested_user.address_1,
                         interested_user.address_2,
-                        interested_user.address_3,
                         interested_user.postal_code,
                         interested_user.country
                     )

microsetta_private_api/api/microsetta_private_api.yaml

@@ -1979,6 +1979,7 @@ paths:
                     type: string
                   'address_2':
                     type: string
+                    nullable: true
                   'city':
                     type: string
                   'state':

microsetta_private_api/repo/campaign_repo.py

@@ -1,6 +1,7 @@
 import psycopg2
 import json
 import datetime
+import uuid
 
 from microsetta_private_api.client.fundrazr import FundrazrClient
 from microsetta_private_api.repo.base_repo import BaseRepo
@@ -10,6 +11,9 @@
 from microsetta_private_api.tasks import send_email
 from microsetta_private_api.config_manager import SERVER_CONFIG
 from microsetta_private_api.localization import EN_US
+from microsetta_private_api.model.log_event import LogEvent
+from microsetta_private_api.repo.event_log_repo import EventLogRepo
+from microsetta_private_api.admin.email_templates import EmailMessage
 
 
 class UnknownItem(psycopg2.errors.ForeignKeyViolation):
@@ -479,11 +483,19 @@ def add_transaction(self, payment):
                 try:
                     # TODO - will need to add actual language flag to the email
                     # Fundrazr doesn't provide a language flag, defer for now
+                    template = "address_invalid"
+                    email_args = {"contact_name": cn,
+                                  "resolution_url": resolution_url}
+
                     send_email(payment.contact_email,
-                               "address_invalid",
-                               {"contact_name": cn,
-                                "resolution_url": resolution_url},
+                               template,
+                               email_args,
                                EN_US)
+
+                    # Log the email being sent
+                    self._log_email(
+                        template, payment.contact_email, email_args
+                    )
                 except:  # noqa
                     # try our best to email
                     pass
@@ -786,3 +798,21 @@ def _payments_from_transactions(self, ids):
                 entry['fundrazr_perks'] = fundrazr_data.get(entry['id'])
 
         return [payment_from_db(data) for data in trn_data]
+
+    def _log_email(self, template, email_address, email_args):
+        # Log the event of the email being sent
+        template_info = EmailMessage[template]
+        event = LogEvent(
+            uuid.uuid4(),
+            template_info.event_type,
+            template_info.event_subtype,
+            None,
+            {
+                # account_id and email are necessary to allow searching the
+                # event log.
+                "account_id": None,
+                "email": email_address,
+                "template": template,
+                "template_args": email_args
+            })
+        EventLogRepo(self._transaction).add_event(event)

microsetta_private_api/repo/melissa_repo.py

@@ -45,8 +45,7 @@ def create_record(self, address_1, address_2, address_3, city, state,
             else:
                 return record_id
 
-    def check_duplicate(self, address_1, address_2, address_3, postal,
-                        country):
+    def check_duplicate(self, address_1, address_2, postal, country):
         """
         Check if an address has already been verified to avoid duplicate
             queries against the Melissa API
@@ -59,7 +58,6 @@ def check_duplicate(self, address_1, address_2, address_3, postal,
         ----------
         address_1 - Primary street address
         address_2 - Secondary street address
-        address_3 - Tertiary street address
         postal - Postal code
         country - Country
 
@@ -69,21 +67,30 @@ def check_duplicate(self, address_1, address_2, address_3, postal,
         Full table row is record is a duplicate
         """
         with self._transaction.dict_cursor() as cur:
-            cur.execute("""SELECT * FROM campaign.melissa_address_queries
-                            WHERE (source_address_1 = %s
-                            AND source_address_2 = %s
-                            AND source_address_3 = %s
-                            AND source_postal = %s
-                            AND source_country = %s
-                            AND result_processed = true)
-                            OR (result_address_1 = %s
-                            AND result_address_2 = %s
-                            AND result_address_3 = %s
-                            AND result_postal = %s
-                            AND result_country = %s
-                            AND result_processed = true)""",
-                        (address_1, address_2, address_3, postal, country,
-                         address_1, address_2, address_3, postal, country))
+            # We need to gracefully handle a null/None value for address_2 as
+            # psycopg won't automatically handle = NULL vs. IS NULL
+            sql = """SELECT * FROM campaign.melissa_address_queries
+                                WHERE (source_address_1 = %s
+                                AND source_{0}
+                                AND source_postal = %s
+                                AND source_country = %s
+                                AND result_processed = true)
+                                OR (result_address_1 = %s
+                                AND result_{0}
+                                AND result_postal = %s
+                                AND result_country = %s
+                                AND result_processed = true)"""
+            if address_2 is None:
+                address_2_is_null = 'address_2 IS NULL'
+                sql = sql.format(address_2_is_null)
+                arguments = (address_1, postal, country,
+                             address_1, postal, country)
+            else:
+                address_2_is_not_null = 'address_2 = %s'
+                sql = sql.format(address_2_is_not_null)
+                arguments = (address_1, address_2, postal, country,
+                             address_1, address_2, postal, country)
+            cur.execute(sql, arguments)
             row = cur.fetchone()
             if row is None:
                 return False

microsetta_private_api/repo/tests/test_melissa_repo.py

@@ -0,0 +1,131 @@
+import unittest
+
+from microsetta_private_api.repo.transaction import Transaction
+from microsetta_private_api.repo.melissa_repo import MelissaRepo
+
+
+class MelissaRepoTests(unittest.TestCase):
+    def test_create_record(self):
+        with Transaction() as t:
+            mr = MelissaRepo(t)
+            record_id = mr.create_record(
+                "9500 Gilman Dr",
+                "",
+                "",
+                "La Jolla",
+                "CA",
+                "92093",
+                "US"
+            )
+            self.assertNotEqual(record_id, None)
+
+    def test_update_results(self):
+        with Transaction() as t:
+            mr = MelissaRepo(t)
+            record_id = mr.create_record(
+                "9500 Gilman Dr",
+                "",
+                "",
+                "La Jolla",
+                "CA",
+                "92093",
+                "US"
+            )
+
+            obs = mr.update_results(
+                record_id,
+                "http://foo.bar",
+                "RAW_RESULT",
+                "AV24, GS05",
+                True,
+                "9500 Gilman Dr, La Jolla, CA 92093, US",
+                "9500 Gilman Dr",
+                "",
+                "",
+                "La Jolla",
+                "CA",
+                "92093",
+                "US",
+                32.8798916,
+                -117.2363115
+            )
+
+            self.assertTrue(obs)
+
+    def test_check_duplicate_no_match(self):
+        with Transaction() as t:
+            mr = MelissaRepo(t)
+            obs = mr.check_duplicate(
+                "1234 Not Real St",
+                "",
+                "99999",
+                "US"
+            )
+
+            self.assertFalse(obs)
+
+    def test_check_duplicate_with_address_2(self):
+        with Transaction() as t:
+            mr = MelissaRepo(t)
+            record_id = mr.create_record(
+                "9500 Gilman Dr",
+                "Suite 100",
+                "",
+                "La Jolla",
+                "CA",
+                "92093",
+                "US"
+            )
+
+            # We need the result_processed column to be TRUE for the dupe check
+            cur = t.cursor()
+            cur.execute(
+                "UPDATE campaign.melissa_address_queries "
+                "SET result_processed = TRUE "
+                "WHERE melissa_address_query_id = %s",
+                (record_id, )
+            )
+
+            obs = mr.check_duplicate(
+                "9500 Gilman Dr",
+                "Suite 100",
+                "92093",
+                "US"
+            )
+
+            self.assertEqual(record_id, obs['melissa_address_query_id'])
+
+    def test_check_duplicate_no_address_2(self):
+        with Transaction() as t:
+            mr = MelissaRepo(t)
+            record_id = mr.create_record(
+                "9500 Gilman Dr",
+                None,
+                None,
+                "La Jolla",
+                "CA",
+                "92093",
+                "US"
+            )
+
+            # We need the result_processed column to be TRUE for the dupe check
+            cur = t.cursor()
+            cur.execute(
+                "UPDATE campaign.melissa_address_queries "
+                "SET result_processed = TRUE "
+                "WHERE melissa_address_query_id = %s",
+                (record_id, )
+            )
+
+            obs = mr.check_duplicate(
+                "9500 Gilman Dr",
+                None,
+                "92093",
+                "US"
+            )
+
+            self.assertEqual(record_id, obs['melissa_address_query_id'])
+
+
+if __name__ == '__main__':
+    unittest.main()

microsetta_private_api/util/melissa.py

@@ -7,6 +7,14 @@
 from microsetta_private_api.config_manager import SERVER_CONFIG
 from microsetta_private_api.exceptions import RepoException
 
+# The response codes we can treat as deliverable
+GOOD_CODES = ["AV25", "AV24", "AV23", "AV22", "AV21"]
+# NB: We're adding "AV14" as a good code but ONLY if there are no error codes.
+# This code reflects an inability to verify at the highest resolution, but we
+# have determined that for certain scenarios like Mail Boxes Etc and similar
+# locations, it's appropriate to treat as good.
+GOOD_CODES_NO_ERROR = ["AV14"]
+
 
 def verify_address(address_1, address_2=None, address_3=None, city=None,
                    state=None, postal=None, country=None):
@@ -24,13 +32,10 @@ def verify_address(address_1, address_2=None, address_3=None, city=None,
         raise KeyError("Must include address_1, postal, and country fields")
 
     with Transaction() as t:
-        # The response codes we can treat as deliverable
-        GOOD_CODES = ["AV25", "AV24", "AV23", "AV22", "AV21"]
-
         melissa_repo = MelissaRepo(t)
 
         dupe_status = melissa_repo.check_duplicate(address_1, address_2,
-                                                   address_3, postal, country)
+                                                   postal, country)
 
         if dupe_status is not False:
             # duplicate record - return result with an added field noting dupe
@@ -65,12 +70,16 @@ def verify_address(address_1, address_2=None, address_3=None, city=None,
                           "ctry": country}
 
             # Melissa API behaves oddly if it receives null values for a2
-            # and a3 - don't send if we don't have actual data for them
+            # and a3, convert to "" if necessary
             if address_2 is not None:
                 url_params["a2"] = address_2
+            else:
+                url_params["a2"] = ""
 
             if address_3 is not None:
                 url_params["a3"] = address_3
+            else:
+                url_params["a3"] = ""
 
             url = SERVER_CONFIG["melissa_url"] + "?%s" % \
                 urllib.parse.urlencode(url_params)
@@ -97,13 +106,22 @@ def verify_address(address_1, address_2=None, address_3=None, city=None,
                 r_formatted_address = record_obj["FormattedAddress"]
                 r_codes = record_obj["Results"]
                 r_good = False
+                r_errors_present = False
+                r_good_conditional = False
 
                 codes = r_codes.split(",")
                 for code in codes:
+                    if code[0:2] == "AE":
+                        r_errors_present = True
+                    if code in GOOD_CODES_NO_ERROR:
+                        r_good_conditional = True
                     if code in GOOD_CODES:
                         r_good = True
                         break
 
+                if r_good_conditional and not r_errors_present:
+                    r_good = True
+
                 r_address_1 = record_obj["AddressLine1"]
                 r_address_2 = record_obj["AddressLine2"]
                 r_address_3 = record_obj["AddressLine3"]