Merge pull request #70 from meaningfy-ws/feature/TED-213

Added SPARQL CLI-tool

setup.py

@@ -75,6 +75,7 @@ def open_local(paths, mode="r", encoding="utf8"):
     packages=packages,
     entry_points={
         "console_scripts": [
+            "sparql_generator = ted_sws.mapping_suite_processor.entrypoints.cmd_sparql_generator:main",
             "yarrrml2rml_converter = ted_sws.mapping_suite_processor.entrypoints.cmd_yarrrml2rml_converter:main",
             "transformer = ted_sws.notice_transformer.entrypoints.cmd_mapping_suite_transformer:main",
             "normalisation_resource_generator = ted_sws.data_manager.entrypoints.generate_mapping_resources:main"

ted_sws/core/adapters/cmd_runner.py

@@ -84,6 +84,16 @@ def run(self):
     def run_cmd(self):
         pass
 
+    def run_cmd_result(self, error: Exception, msg: str = None, errmsg: str = None) -> bool:
+        if error:
+            self.log_failed_error(error)
+            if errmsg is not None:
+                self.log_failed_msg(errmsg)
+            return False
+        else:
+            self.log_success_msg(msg)
+            return True
+
     def on_end(self):
         self.end_time = datetime.datetime.now()
         self.log("CMD :: END :: {now} :: [{time}]".format(

ted_sws/mapping_suite_processor/entrypoints/cmd_sparql_generator.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python3
+
+import os
+from pathlib import Path
+
+import click
+
+from ted_sws.core.adapters.cmd_runner import CmdRunner as BaseCmdRunner
+from ted_sws.core.adapters.logger import LOG_INFO_TEXT
+from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_sparql_queries import \
+    mapping_suite_processor_generate_sparql_queries as generate_sparql_queries, DEFAULT_RQ_NAME
+
+DEFAULT_MAPPINGS_PATH = 'mappings'
+DEFAULT_CONCEPTUAL_MAPPINGS_FILE = '{mappings_path}/{mapping_suite_id}/transformation/conceptual_mappings.xlsx'
+DEFAULT_OUTPUT_SPARQL_QUERIES_FOLDER = '{mappings_path}/{mapping_suite_id}/validation/sparql/cm_assertions'
+CMD_NAME = "CMD_SPARQL_GENERATOR"
+
+"""
+USAGE:
+# sparql_generator --help
+"""
+
+
+class CmdRunner(BaseCmdRunner):
+    """
+    Keeps the logic to be used by SPARQL queries Generator
+    """
+
+    def __init__(
+            self,
+            conceptual_mappings_file,
+            output_sparql_queries_folder,
+            rq_name
+    ):
+        super().__init__(name=CMD_NAME)
+        self.conceptual_mappings_file_path = Path(os.path.realpath(conceptual_mappings_file))
+        self.output_sparql_queries_folder_path = Path(os.path.realpath(output_sparql_queries_folder))
+        self.rq_name = rq_name
+
+        if not self.conceptual_mappings_file_path.is_file():
+            error_msg = "No such file :: [" + conceptual_mappings_file + "]"
+            self.log_failed_msg(error_msg)
+            raise FileNotFoundError(error_msg)
+
+    def run_cmd(self):
+        self.generate(self.conceptual_mappings_file_path, self.output_sparql_queries_folder_path, self.rq_name)
+
+    def generate(self, conceptual_mappings_file_path, rml_output_file_path, rq_name):
+        """
+        Generates SPARQL queries from Conceptual Mappings
+        """
+        self.log("Running " + LOG_INFO_TEXT.format("SPARQL queries") + " generation ... ")
+
+        error = None
+        try:
+            generate_sparql_queries(conceptual_mappings_file_path, rml_output_file_path, rq_name)
+        except Exception as e:
+            error = e
+
+        return self.run_cmd_result(error)
+
+
+@click.command()
+@click.argument('mapping-suite-id', nargs=1, required=False)
+@click.option('-i', '--opt-conceptual-mappings-file', help="Use to overwrite INPUT generator")
+@click.option('-o', '--opt-output-sparql-queries-folder', help="Use to overwrite OUTPUT generator")
+@click.option('-rq-name', '--opt-rq-name', default=DEFAULT_RQ_NAME)
+@click.option('-m', '--opt-mappings-path', default=DEFAULT_MAPPINGS_PATH)
+def main(mapping_suite_id, opt_conceptual_mappings_file, opt_output_sparql_queries_folder,
+         opt_rq_name, opt_mappings_path):
+    """
+    Generates SPARQL queries from Conceptual Mappings.
+    """
+
+    if opt_conceptual_mappings_file:
+        conceptual_mappings_file = opt_conceptual_mappings_file
+    else:
+        conceptual_mappings_file = DEFAULT_CONCEPTUAL_MAPPINGS_FILE.format(
+            mappings_path=opt_mappings_path,
+            mapping_suite_id=mapping_suite_id
+        )
+
+    if opt_output_sparql_queries_folder:
+        output_sparql_queries_folder = opt_output_sparql_queries_folder
+    else:
+        output_sparql_queries_folder = DEFAULT_OUTPUT_SPARQL_QUERIES_FOLDER.format(
+            mappings_path=opt_mappings_path,
+            mapping_suite_id=mapping_suite_id
+        )
+
+    cmd = CmdRunner(
+        conceptual_mappings_file=conceptual_mappings_file,
+        output_sparql_queries_folder=output_sparql_queries_folder,
+        rq_name=opt_rq_name
+    )
+    cmd.run()
+
+
+if __name__ == '__main__':
+    main()

ted_sws/mapping_suite_processor/entrypoints/cmd_yarrrml2rml_converter.py

@@ -60,12 +60,7 @@ def convert(self, yarrrml_input_file_path, rml_output_file_path):
         except Exception as e:
             error = e
 
-        if error:
-            self.log_failed_error(error)
-            return False
-        else:
-            self.log_success_msg()
-            return True
+        return self.run_cmd_result(error)
 
 
 @click.command()

ted_sws/mapping_suite_processor/services/conceptual_mapping_generate_sparql_queries.py

@@ -12,6 +12,8 @@
 RULES_CLASS_PATH = 'Class path (M)'
 RULES_PROPERTY_PATH = 'Property path (M)'
 
+DEFAULT_RQ_NAME = 'sparql_query_'
+
 
 def sparql_validation_generator(data: pd.DataFrame) -> Iterator[str]:
     """
@@ -34,11 +36,13 @@ def sparql_validation_generator(data: pd.DataFrame) -> Iterator[str]:
 
 
 def mapping_suite_processor_generate_sparql_queries(conceptual_mappings_file_path: pathlib.Path,
-                                                    output_sparql_queries_folder_path: pathlib.Path):
+                                                    output_sparql_queries_folder_path: pathlib.Path,
+                                                    rq_name: str = DEFAULT_RQ_NAME):
     """
         This function reads data from conceptual_mappings.xlsx and generates SPARQL validation queries in provided package.
     :param conceptual_mappings_file_path:
     :param output_sparql_queries_folder_path:
+    :param rq_name:
     :return:
     """
     with open(conceptual_mappings_file_path, 'rb') as excel_file:
@@ -48,7 +52,8 @@ def mapping_suite_processor_generate_sparql_queries(conceptual_mappings_file_pat
         conceptual_mappings_rules_df = conceptual_mappings_rules_df[
             conceptual_mappings_rules_df[RULES_PROPERTY_PATH].notnull()]
     sparql_queries = sparql_validation_generator(conceptual_mappings_rules_df)
+    output_sparql_queries_folder_path.mkdir(parents=True, exist_ok=True)
     for index, sparql_query in enumerate(sparql_queries):
-        output_file_path = output_sparql_queries_folder_path / f"sparql_query_{index}.rq"
+        output_file_path = output_sparql_queries_folder_path / f"{rq_name}{index}.rq"
         with open(output_file_path, "w") as output_file:
             output_file.write(sparql_query)

ted_sws/notice_transformer/entrypoints/cmd_mapping_suite_transformer.py

@@ -86,14 +86,8 @@ def transform(self, mapping_suite_id, serialization_format_value):
         except Exception as e:
             error = e
 
-        suite_text = mapping_suite_id
-        if error:
-            self.log_failed_error(error)
-            self.log_failed_msg(suite_text)
-            return False
-        else:
-            self.log_success_msg(suite_text)
-            return True
+        msg = mapping_suite_id
+        return self.run_cmd_result(error, msg, msg)
 
 
 @click.command()

tests/unit/mapping_suite_processor/test_cmd_sparql_generator.py

@@ -0,0 +1,37 @@
+import os
+
+from click.testing import CliRunner
+
+from ted_sws.mapping_suite_processor.entrypoints.cmd_sparql_generator import main as generate
+
+cmdRunner = CliRunner()
+
+
+def __process_output_dir(fake_repository_path, fake_mapping_suite_id):
+    output_dir_path = fake_repository_path / fake_mapping_suite_id / "validation" / "sparql" / "cm_assertions"
+    assert os.path.isdir(output_dir_path)
+    for filename in os.listdir(output_dir_path):
+        f = os.path.join(output_dir_path, filename)
+        if os.path.isfile(f):
+            os.remove(f)
+
+
+def test_sparql_generator(fake_mapping_suite_id, file_system_repository_path):
+    response = cmdRunner.invoke(generate, [fake_mapping_suite_id, "--opt-mappings-path", file_system_repository_path])
+    assert response.exit_code == 0
+    assert "SUCCESS" in response.output
+    __process_output_dir(file_system_repository_path, fake_mapping_suite_id)
+
+
+def test_sparql_generator_with_non_existing_input(file_system_repository_path):
+    response = cmdRunner.invoke(generate, ["-i", "non_existing_dir/non_existing_file",
+                                           "-o", "non_existing_dir",
+                                           "--opt-mappings-path", file_system_repository_path])
+    assert "No such file" in response.output
+
+
+def test_sparql_generator_with_invalid_input(file_system_repository_path, fake_mapping_suite_id):
+    response = cmdRunner.invoke(generate, ["-i", str(file_system_repository_path / fake_mapping_suite_id /
+                                                     "transformation" / "invalid_conceptual_mappings.xlsx"),
+                                           "--opt-mappings-path", file_system_repository_path])
+    assert "FAILED" in response.output

tests/unit/mapping_suite_processor/test_cmd_yarrrml2rml_converter.py

@@ -22,13 +22,13 @@ def test_cmd_converter(fake_mapping_suite_id, file_system_repository_path):
     __process_output_dir(file_system_repository_path, fake_mapping_suite_id)
 
 
-def test_cmd_converter_with_invalid_output(fake_mapping_suite_id, file_system_repository_path):
+def test_cmd_converter_with_non_existing_output(fake_mapping_suite_id, file_system_repository_path):
     response = cmdRunner.invoke(convert, [fake_mapping_suite_id, "-o", "non_existing_dir/non_existing_file",
                                           "--opt-mappings-path", file_system_repository_path])
     assert "FAILED" in response.output
 
 
-def test_cmd_converter_with_invalid_input(file_system_repository_path):
+def test_cmd_converter_with_non_existing_input(file_system_repository_path):
     response = cmdRunner.invoke(convert, ["-i", "non_existing_dir/non_existing_file",
                                           "-o", "non_existing_dir/non_existing_file",
                                           "--opt-mappings-path", file_system_repository_path])