Feature/ted 211

ted_sws/mapping_suite_processor/services/conceptual_mapping_files_injection.py

@@ -1,5 +1,6 @@
 import pathlib
 import shutil
+
 import pandas as pd
 
 CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME = "Resources"
@@ -20,7 +21,7 @@ def mapping_suite_processor_inject_resources(conceptual_mappings_file_path: path
     :return:
     """
     resources_df = pd.read_excel(conceptual_mappings_file_path,
-                                         sheet_name=CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME)
+                                 sheet_name=CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME)
     resource_file_names = list(resources_df[FILE_NAME_KEY].values)
     for resource_file_name in resource_file_names:
         src_resource_file_path = resources_folder_path / resource_file_name
@@ -29,13 +30,25 @@ def mapping_suite_processor_inject_resources(conceptual_mappings_file_path: path
 
 
 def mapping_suite_processor_inject_shacl_shapes(shacl_shape_file_path: pathlib.Path,
-                                                output_resources_folder_path: pathlib.Path):
+                                                output_shacl_shape_folder_path: pathlib.Path):
     """
         This function copies a shacl_shape file to the desired directory.
-    :param conceptual_mappings_file_path:
     :param shacl_shape_file_path:
-    :param output_resources_folder_path:
+    :param output_shacl_shape_folder_path:
     :return:
     """
-    dest_shacl_shape_file_path = output_resources_folder_path / shacl_shape_file_path.name
+    dest_shacl_shape_file_path = output_shacl_shape_folder_path / shacl_shape_file_path.name
     shutil.copy(shacl_shape_file_path, dest_shacl_shape_file_path)
+
+
+def mapping_suite_processor_inject_sparql_queries(sparql_queries_folder_path: pathlib.Path,
+                                                  output_sparql_queries_folder_path: pathlib.Path
+                                                  ):
+    """
+       This function copies SPARQL queries from the source folder to the destination folder.
+    :param sparql_queries_folder_path:
+    :param output_sparql_queries_folder_path:
+    :return:
+    """
+    shutil.copytree(sparql_queries_folder_path, output_sparql_queries_folder_path)
+

ted_sws/mapping_suite_processor/services/conceptual_mapping_processor.py

@@ -1,9 +1,14 @@
 import pathlib
+import shutil
+
+from pymongo import MongoClient
 
 from ted_sws.data_manager.adapters.mapping_suite_repository import TRANSFORM_PACKAGE_NAME, VALIDATE_PACKAGE_NAME, \
-    SPARQL_PACKAGE_NAME, METADATA_FILE_NAME, RESOURCES_PACKAGE_NAME, SHACL_PACKAGE_NAME
+    SPARQL_PACKAGE_NAME, METADATA_FILE_NAME, RESOURCES_PACKAGE_NAME, SHACL_PACKAGE_NAME, TEST_DATA_PACKAGE_NAME, \
+    MappingSuiteRepositoryInFileSystem, MappingSuiteRepositoryMongoDB
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_files_injection import \
-    mapping_suite_processor_inject_resources, mapping_suite_processor_inject_shacl_shapes
+    mapping_suite_processor_inject_resources, mapping_suite_processor_inject_shacl_shapes, \
+    mapping_suite_processor_inject_sparql_queries
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_metadata import \
     mapping_suite_processor_generate_metadata
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_generate_sparql_queries import \
@@ -16,6 +21,28 @@
 SHACL_SHAPE_RESOURCES_FOLDER = "shacl_shapes"
 SHACL_SHAPE_FILE_NAME = "ePO_shacl_shapes.rdf"
 MAPPING_FILES_RESOURCES_FOLDER = "mapping_files"
+SPARQL_QUERIES_RESOURCES_FOLDER = "queries"
+SPARQL_QUERIES_INJECTION_FOLDER = "business_queries"
+PROD_ARCHIVE_SUFFIX = "prod"
+DEMO_ARCHIVE_SUFFIX = "demo"
+
+
+def mapping_suite_processor_zip_package(mapping_suite_package_path: pathlib.Path,
+                                        prod_version: bool = False):
+    """
+        This function archives a package and puts a suffix in the name of the archive.
+    :param mapping_suite_package_path:
+    :param prod_version:
+    :return:
+    """
+    archive_name_suffix = PROD_ARCHIVE_SUFFIX if prod_version else DEMO_ARCHIVE_SUFFIX
+    tmp_folder_path = mapping_suite_package_path.parent / f"{mapping_suite_package_path.stem}-{archive_name_suffix}"
+    output_archive_file_name = mapping_suite_package_path.parent / f"{mapping_suite_package_path.stem}-{archive_name_suffix}"
+    shutil.copytree(mapping_suite_package_path, tmp_folder_path)
+    if prod_version:
+        shutil.rmtree(tmp_folder_path / TEST_DATA_PACKAGE_NAME)
+    shutil.make_archive(str(output_archive_file_name), 'zip', tmp_folder_path)
+    shutil.rmtree(tmp_folder_path)
 
 
 def mapping_suite_processor_expand_package(mapping_suite_package_path: pathlib.Path):
@@ -31,6 +58,8 @@ def mapping_suite_processor_expand_package(mapping_suite_package_path: pathlib.P
     mapping_files_resources_folder_path = RESOURCES_PATH / MAPPING_FILES_RESOURCES_FOLDER
     shacl_shape_file_path = RESOURCES_PATH / SHACL_SHAPE_RESOURCES_FOLDER / SHACL_SHAPE_FILE_NAME
     shacl_shape_injection_folder = mapping_suite_package_path / VALIDATE_PACKAGE_NAME / SHACL_PACKAGE_NAME / SHACL_SHAPE_INJECTION_FOLDER
+    sparql_queries_resources_folder_path = RESOURCES_PATH / SPARQL_QUERIES_RESOURCES_FOLDER
+    sparql_queries_injection_folder = mapping_suite_package_path / VALIDATE_PACKAGE_NAME / SPARQL_PACKAGE_NAME / SPARQL_QUERIES_INJECTION_FOLDER
     shacl_shape_injection_folder.mkdir(parents=True, exist_ok=True)
     cm_sparql_folder_path.mkdir(parents=True, exist_ok=True)
     resources_folder_path.mkdir(parents=True, exist_ok=True)
@@ -49,5 +78,28 @@ def mapping_suite_processor_expand_package(mapping_suite_package_path: pathlib.P
                                              )
 
     mapping_suite_processor_inject_shacl_shapes(shacl_shape_file_path=shacl_shape_file_path,
-                                                output_resources_folder_path = shacl_shape_injection_folder
+                                                output_shacl_shape_folder_path=shacl_shape_injection_folder
                                                 )
+    mapping_suite_processor_inject_sparql_queries(sparql_queries_folder_path=sparql_queries_resources_folder_path,
+                                                  output_sparql_queries_folder_path=sparql_queries_injection_folder
+                                                  )
+
+    mapping_suite_processor_zip_package(mapping_suite_package_path=mapping_suite_package_path)
+    mapping_suite_processor_zip_package(mapping_suite_package_path=mapping_suite_package_path, prod_version=True)
+
+
+def mapping_suite_processor_load_package_in_mongo_db(mapping_suite_package_path: pathlib.Path,
+                                                     mongodb_client: MongoClient):
+    """
+        This feature allows you to upload a mapping suite package to MongoDB.
+    :param mapping_suite_package_path:
+    :param mongodb_client:
+    :return:
+    """
+    mapping_suite_repository_path = mapping_suite_package_path.parent
+    mapping_suite_package_name = mapping_suite_package_path.name
+    mapping_suite_repository_in_file_system = MappingSuiteRepositoryInFileSystem(
+        repository_path=mapping_suite_repository_path)
+    mapping_suite_in_memory = mapping_suite_repository_in_file_system.get(reference=mapping_suite_package_name)
+    mapping_suite_repository_mongo_db = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+    mapping_suite_repository_mongo_db.add(mapping_suite=mapping_suite_in_memory)

tests/unit/mapping_suite_processor/conftest.py

@@ -1,3 +1,5 @@
+import mongomock
+import pymongo
 import pytest
 
 from tests import TEST_DATA_PATH
@@ -6,3 +8,8 @@
 @pytest.fixture
 def file_system_repository_path():
     return TEST_DATA_PATH / "notice_transformer" / "mapping_suite_processor_repository"
+
+@pytest.fixture
+@mongomock.patch(servers=(('server.example.com', 27017),))
+def mongodb_client():
+    return pymongo.MongoClient('server.example.com')

tests/unit/mapping_suite_processor/test_conceptual_mapping_processor.py

@@ -1,6 +1,7 @@
-from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem
+from ted_sws.data_manager.adapters.mapping_suite_repository import MappingSuiteRepositoryInFileSystem, \
+    MappingSuiteRepositoryMongoDB
 from ted_sws.mapping_suite_processor.services.conceptual_mapping_processor import CONCEPTUAL_MAPPINGS_ASSERTIONS, \
-    mapping_suite_processor_expand_package
+    mapping_suite_processor_expand_package, mapping_suite_processor_load_package_in_mongo_db
 from tests import temporary_copy
 
 
@@ -13,7 +14,6 @@ def test_mapping_suite_processor_expand_package(file_system_repository_path):
         mapping_suite = mapping_suite_repository.get(reference="test_package")
         assert mapping_suite
         assert mapping_suite.sparql_test_suites
-        assert len(mapping_suite.sparql_test_suites) == 1
         sparql_packages = set(map(lambda x: x.identifier, mapping_suite.sparql_test_suites))
         assert CONCEPTUAL_MAPPINGS_ASSERTIONS in sparql_packages
         for sparql_test_suite in mapping_suite.sparql_test_suites:
@@ -22,7 +22,7 @@ def test_mapping_suite_processor_expand_package(file_system_repository_path):
 
         assert mapping_suite.metadata_constraints
         assert mapping_suite.title == "sample_title"
-        assert mapping_suite.identifier == "mapping_id"
+        assert mapping_suite.identifier == "test_package"
         assert mapping_suite.version == "0.0.1"
         assert mapping_suite.ontology_version == "3.0.0.alpha"
         assert "F03" in set(mapping_suite.metadata_constraints.constraints["form_number"])
@@ -33,3 +33,37 @@ def test_mapping_suite_processor_expand_package(file_system_repository_path):
         assert mapping_suite.transformation_rule_set.resources
         assert len(mapping_suite.transformation_rule_set.resources) == 3
 
+        assert mapping_suite.sparql_test_suites
+        assert len(mapping_suite.sparql_test_suites) == 2
+        assert "business_queries" in set(map(lambda x: x.identifier, mapping_suite.sparql_test_suites))
+        assert "cm_assertions" in set(map(lambda x: x.identifier, mapping_suite.sparql_test_suites))
+        assert "not_cm_assertions" not in set(map(lambda x: x.identifier, mapping_suite.sparql_test_suites))
+
+        tmp_prod_archive_path = tmp_mapping_suite_package_path.parent / f"{tmp_mapping_suite_package_path.stem}-prod.zip"
+        tmp_demo_archive_path = tmp_mapping_suite_package_path.parent / f"{tmp_mapping_suite_package_path.stem}-demo.zip"
+
+        assert tmp_prod_archive_path.is_file()
+        assert tmp_demo_archive_path.is_file()
+
+        tmp_prod_archive_path.unlink()
+        tmp_demo_archive_path.unlink()
+
+        assert not tmp_prod_archive_path.is_file()
+        assert not tmp_demo_archive_path.is_file()
+
+
+def test_mapping_suite_processor_upload_in_mongodb(file_system_repository_path, mongodb_client):
+    with temporary_copy(file_system_repository_path) as tmp_mapping_suite_package_path:
+        mapping_suite_package_path = tmp_mapping_suite_package_path / "test_package"
+        mapping_suite_processor_expand_package(mapping_suite_package_path=mapping_suite_package_path)
+        mapping_suite_processor_load_package_in_mongo_db(mapping_suite_package_path=mapping_suite_package_path,
+                                                          mongodb_client=mongodb_client)
+        mapping_suite_repository = MappingSuiteRepositoryInFileSystem(
+            repository_path=tmp_mapping_suite_package_path)
+        mapping_suite = mapping_suite_repository.get(reference=mapping_suite_package_path.name)
+        assert mapping_suite
+        mapping_suite_repository = MappingSuiteRepositoryMongoDB(mongodb_client=mongodb_client)
+        mapping_suite = mapping_suite_repository.get(reference=mapping_suite_package_path.name)
+        assert mapping_suite
+
+    mongodb_client.drop_database(MappingSuiteRepositoryMongoDB._database_name)