Merge branch 'skip_wrong_args' of 'https://github.com/zhquan/GrimoireELK

'

Merges #1053
Closes #1053

grimoire_elk/elk.py

@@ -203,8 +203,18 @@ def feed_backend(url, clean, fetch_archive, backend_name, backend_params,
         else:
             error_msg = "Error feeding raw from {}".format(ex)
             logger.error(error_msg, exc_info=True)
+    except SystemExit:
+        anonymized_params = anonymize_params(backend_params)
+        msg = "Wrong {} arguments: {}".format(backend_name, anonymized_params)
+        error_msg = "Error feeding raw. {}".format(msg)
+        logger.error(error_msg, exc_info=True)
+
+    try:
+        msg = "[{}] Done collection for {}".format(backend_name, anonymize_url(backend.origin))
+    except AttributeError:
+        msg = "[{}] Done collection for {}".format(backend_name, anonymize_url(projects_json_repo))
+    logger.info(msg)
 
-    logger.info("[{}] Done collection for {}".format(backend_name, anonymize_url(backend.origin)))
     return error_msg
 
 
@@ -603,8 +613,18 @@ def enrich_backend(url, clean, backend_name, backend_params, cfg_section_name,
                          backend_name, anonymize_url(backend.origin), ex), exc_info=True)
         else:
             logger.error("Error enriching raw {}".format(ex), exc_info=True)
+    except SystemExit:
+        anonymized_params = anonymize_params(backend_params)
+        msg = "Wrong {} arguments: {}".format(backend_name, anonymized_params)
+        error_msg = "Error enriching raw. {}".format(msg)
+        logger.error(error_msg, exc_info=True)
+
+    try:
+        msg = "[{}] Done enrichment for {}".format(backend_name, anonymize_url(backend.origin))
+    except AttributeError:
+        msg = "[{}] Done enrichment for {}".format(backend_name, anonymize_url(projects_json_repo))
 
-    logger.info("[{}] Done enrichment for {}".format(backend_name, anonymize_url(backend.origin)))
+    logger.info(msg)
 
 
 def delete_orphan_unique_identities(es, sortinghat_db, current_data_source, active_data_sources):

releases/unreleased/skip-sources-with-wrong-arguments.yml

@@ -0,0 +1,8 @@
+---
+title: Skip sources with wrong arguments
+category: fixed
+author: Quan Zhou <[email protected]>
+issue: null
+notes: >
+    Sources won't interrupt collection or enrichment processes when any
+    of their arguments are invalid. Instead, they will be skipped.

tests/test_elk.py

@@ -21,9 +21,10 @@
 #
 
 import configparser
+import logging
 import unittest
 
-from grimoire_elk.elk import anonymize_params
+from grimoire_elk.elk import anonymize_params, enrich_backend, feed_backend, logger
 
 
 CONFIG_FILE = 'tests.conf'
@@ -50,3 +51,51 @@ def test_anonymize_params(self):
         params = ("--backend-password", "mypassword", "--no-archive", "--api-token", "token")
         anonymized_params = anonymize_params(params)
         self.assertTupleEqual(anonymized_params, expected_params)
+
+    def test_feed_backend_wrong_params(self):
+        """Test feed_backend with wrong parameters."""
+
+        # Jira has not --api-token argument
+        backend_params = ['https://jira.example.org', '--api-token', 'mypersonaltoken']
+        projects_json_repo = 'https://jira.example.org'
+        expected_msg = "Error feeding raw. Wrong jira arguments: ('https://jira.example.org', " \
+                       "'--api-token', 'xxxxx')"
+        error_msg = feed_backend(self.es_con, False, False, "jira", backend_params,
+                                 "jira_raw", "jira_enriched", projects_json_repo=projects_json_repo)
+        self.assertEqual(error_msg, expected_msg)
+
+        # Github the repository name cannot start with '-'
+        backend_params = ['chaoss', '-grimoirelab-elk', '--api-token', 'mypersonaltoken']
+        projects_json_repo = 'https://github.com/chaoss/-grimoirelab-elk'
+        expected_msg = "Error feeding raw. Wrong github arguments: ('chaoss', " \
+                       "'-grimoirelab-elk', '--api-token', 'xxxxx')"
+        error_msg = feed_backend(self.es_con, False, False, "github", backend_params,
+                                 "github_raw", "github_enriched", projects_json_repo=projects_json_repo)
+        self.assertEqual(error_msg, expected_msg)
+
+    def test_enrich_backend_wrong_params(self):
+        """Test enrich_backend with wrong parameters."""
+
+        # Jira has not --api-token argument
+        backend_params = ['https://jira.example.org', '--api-token', 'mypersonaltoken']
+        projects_json_repo = 'https://jira.example.org'
+        expected_msg = "Error enriching raw. Wrong jira arguments: ('https://jira.example.org', '--api-token', 'xxxxx')"
+        with self.assertLogs(logger, level='ERROR') as cm:
+            enrich_backend(self.es_con, False, "jira", backend_params, "jira",
+                           "jira_raw", "jira_enriched", projects_json_repo=projects_json_repo)
+            self.assertEqual(cm.records[0].msg, expected_msg)
+
+        # Github the repository name cannot start with '-'
+        backend_params = ['chaoss', '-grimoirelab-elk', '--api-token', 'mypersonaltoken']
+        projects_json_repo = 'https://github.com/chaoss/-grimoirelab-elk'
+        expected_msg = "Error enriching raw. Wrong github arguments: ('chaoss', " \
+                       "'-grimoirelab-elk', '--api-token', 'xxxxx')"
+        with self.assertLogs(logger, level='ERROR') as cm:
+            enrich_backend(self.es_con, False, "github", backend_params, "github",
+                           "github_raw", "github_enriched", projects_json_repo=projects_json_repo)
+            self.assertEqual(cm.records[0].msg, expected_msg)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    unittest.main()