From 5e5a03cf524e1ec309e9741fbe83564e97a9bf79 Mon Sep 17 00:00:00 2001 From: Animesh Kumar Date: Wed, 15 Apr 2020 22:14:51 +0530 Subject: [PATCH] [metadata] Remove metadata from empty enriched items This commit updates the enricher for github and gitlab to prevent addition of metadata fields to empty enriched items Signed-off-by: Animesh Kumar --- grimoire_elk/enriched/enrich.py | 6 ++++++ grimoire_elk/enriched/github.py | 1 + grimoire_elk/enriched/github2.py | 3 +++ grimoire_elk/enriched/gitlab.py | 3 ++- 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/grimoire_elk/enriched/enrich.py b/grimoire_elk/enriched/enrich.py index b9c036658..15757a5d3 100644 --- a/grimoire_elk/enriched/enrich.py +++ b/grimoire_elk/enriched/enrich.py @@ -91,6 +91,10 @@ def metadata(func): @functools.wraps(func) def decorator(self, *args, **kwargs): eitem = func(self, *args, **kwargs) + + if not eitem: + return eitem + metadata = { 'metadata__gelk_version': self.gelk_version, 'metadata__gelk_backend_name': self.__class__.__name__, @@ -388,6 +392,8 @@ def enrich_items(self, ocean_backend, events=False): if not events: rich_item = self.get_rich_item(item) + if not rich_item: + continue data_json = json.dumps(rich_item) bulk_json += '{"index" : {"_id" : "%s" } }\n' % \ (item[self.get_field_unique_id()]) diff --git a/grimoire_elk/enriched/github.py b/grimoire_elk/enriched/github.py index e51e77970..0e24d2057 100644 --- a/grimoire_elk/enriched/github.py +++ b/grimoire_elk/enriched/github.py @@ -226,6 +226,7 @@ def get_rich_item(self, item): else: logger.error("[github] rich item not defined for GitHub category {}".format( item['category'])) + return rich_item self.add_repository_labels(rich_item) self.add_metadata_filter_raw(rich_item) diff --git a/grimoire_elk/enriched/github2.py b/grimoire_elk/enriched/github2.py index cb10c977a..91ec029c4 100644 --- a/grimoire_elk/enriched/github2.py +++ b/grimoire_elk/enriched/github2.py @@ -228,6 +228,7 @@ def get_rich_item(self, item): else: logger.error("[github] rich item not defined for GitHub category {}".format( item['category'])) + return rich_item self.add_repository_labels(rich_item) self.add_metadata_filter_raw(rich_item) @@ -386,6 +387,8 @@ def enrich_items(self, ocean_backend): eitems = [] eitem = self.get_rich_item(item) + if not eitem: + continue items_to_enrich.append(eitem) if item['category'] == ISSUE_TYPE: eitems = self.enrich_issue(item, eitem) diff --git a/grimoire_elk/enriched/gitlab.py b/grimoire_elk/enriched/gitlab.py index f5f51a045..ef8eea4f2 100644 --- a/grimoire_elk/enriched/gitlab.py +++ b/grimoire_elk/enriched/gitlab.py @@ -148,7 +148,8 @@ def get_rich_item(self, item): elif item['category'] == 'merge_request': rich_item = self.__get_rich_merge(item) else: - logger.error("[gerrit] rich item not defined for GitLab category {}".format(item['category'])) + logger.error("[gitlab] rich item not defined for GitLab category {}".format(item['category'])) + return rich_item self.add_repository_labels(rich_item) self.add_metadata_filter_raw(rich_item)