[enrich] Avoid "-- UNDEFINED --" values

Avoid `-- UNDEFINED --` values for all SortingHat fields when
Mordred loses connection to the SortingHat server during the
enrichment or autorefresh execution. It will keep the values
of the `name`, `email`, `id` and `uuid` fields.

Signed-off-by: Quan Zhou <[email protected]>

grimoire_elk/enriched/enrich.py

@@ -826,13 +826,16 @@ def get_sh_item_from_identity_cache(self, identity_tuple, backend_name):
                                          email=iden['email'],
                                          name=iden['name'],
                                          username=iden['username'])
+        iden['uuid'] = identity_id
+        iden['id'] = identity_id
+        iden['enrollments'] = []
 
         try:
             individual = self.get_entity(identity_id)
             if not individual:
                 msg = "Individual not found given the following identity: {}".format(identity_id)
                 logger.debug(msg)
-                return sh_item
+                return iden
 
             for indv_identity in individual['identities']:
                 if indv_identity['uuid'] == identity_id:
@@ -842,10 +845,10 @@ def get_sh_item_from_identity_cache(self, identity_tuple, backend_name):
                 msg = "Identity {} not found in individual returned by SortingHat.".format(identity)
                 logger.error(msg)
                 return sh_item
-        except SortingHatClientError:
+        except SortingHatClientError as e:
             msg = "None Identity found {}, identity: {}".format(backend_name, identity)
-            logger.debug(msg)
-            return sh_item
+            logger.error(msg)
+            raise SortingHatClientError(e)
         except UnicodeEncodeError:
             msg = "UnicodeEncodeError {}, identity: {}".format(backend_name, identity)
             logger.error(msg)
@@ -1054,7 +1057,6 @@ def get_item_sh(self, item, roles=None, date_field=None):
                     sh_fields = self.get_item_sh_fields(identity, item_date, rol=rol)
                 else:
                     sh_fields = self.get_item_no_sh_fields(identity, rol)
-
                 eitem_sh.update(sh_fields)
 
                 if not eitem_sh[rol + '_org_name']:

grimoire_elk/enriched/sortinghat_gelk.py

@@ -143,8 +143,11 @@ def add_identities(cls, db, identities, backend):
         except SortingHatClientError as ex:
             for error in ex.errors:
                 msg = error['message']
+                status = error.get('status', None)
                 if 'already exists' in msg:
                     logger.debug("[sortinghat] {}".format(msg))
+                elif status and status == 502:
+                    raise SortingHatClientError(ex)
                 else:
                     logger.warning("[sortinghat] {}".format(msg))
 
@@ -361,6 +364,7 @@ def get_entity(cls, db, id):
                 entity = result['data']['individuals']['entities'][0]
         except SortingHatClientError as e:
             logger.error("[sortinghat] Error get entities {}: {}".format(id, e.errors[0]['message']))
+            raise SortingHatClientError(e)
         return entity
 
     @classmethod

releases/unreleased/avoid-undefined-values-in-author-fields.yml

@@ -0,0 +1,10 @@
+---
+title: Avoid UNDEFINED values in author fields
+category: fixed
+author: Quan Zhou <[email protected]>
+issue: null
+notes: >
+    Avoid `-- UNDEFINED --` values for all SortingHat fields when
+    Mordred loses connection to the SortingHat server during the
+    enrichment or autorefresh execution. It will keep the values
+    of the `name`, `email`, `id` and `uuid` fields.

tests/test_git.py

@@ -493,7 +493,7 @@ def test_onion_study(self):
         url = self.es_con + "/test_git_onion/_search?size=50"
         response = requests.get(url, verify=False).json()
         hits = response['hits']['hits']
-        self.assertEqual(len(hits), 16)
+        self.assertEqual(len(hits), 28)
         for hit in hits:
             source = hit['_source']
             self.assertIn('timeframe', source)

tests/test_github.py

@@ -405,7 +405,7 @@ def test_onion_study(self):
         url = self.es_con + "/test_github_issues_onion/_search?size=20"
         response = requests.get(url, verify=False).json()
         hits = response['hits']['hits']
-        self.assertEqual(len(hits), 10)
+        self.assertEqual(len(hits), 12)
         for hit in hits:
             source = hit['_source']
             self.assertIn('timeframe', source)
@@ -426,7 +426,7 @@ def test_onion_study(self):
         url = self.es_con + "/test_github_prs_onion/_search?size=20"
         response = requests.get(url, verify=False).json()
         hits = response['hits']['hits']
-        self.assertEqual(len(hits), 10)
+        self.assertEqual(len(hits), 12)
         for hit in hits:
             source = hit['_source']
             self.assertIn('timeframe', source)

tests/test_gitlab.py

@@ -314,7 +314,7 @@ def test_onion_study(self):
         url = self.es_con + "/test_gitlab_onion/_search?size=50"
         response = requests.get(url, verify=False).json()
         hits = response['hits']['hits']
-        self.assertEqual(len(hits), 10)
+        self.assertEqual(len(hits), 22)
         for hit in hits:
             source = hit['_source']
             self.assertIn('timeframe', source)