Feature/ted 975

ted_sws/data_manager/adapters/mapping_suite_repository.py

@@ -208,11 +208,13 @@ def _write_file_resources(self, file_resources: List[FileResource], path: pathli
                 f.write(file_resource.file_content)
 
     @classmethod
-    def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, extension=None) -> List[FileResource]:
+    def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, extension=None, with_content=True) -> \
+            List[FileResource]:
         """
         This method reads a folder (with nested-tree structure) of resources and returns a flat list of file-type
         resources from all beyond levels.
         Used for folders that contains files with unique names, but grouped into sub-folders.
+        :param with_content:
         :param extension:
         :param path:
         :param file_resources:
@@ -230,7 +232,8 @@ def read_flat_file_resources(cls, path: pathlib.Path, file_resources=None, exten
                     continue
                 file_path = pathlib.Path(os.path.join(root, f))
                 file_resource = FileResource(file_name=file_path.name,
-                                             file_content=file_path.read_text(encoding="utf-8"),
+                                             file_content=file_path.read_text(
+                                                 encoding="utf-8") if with_content else "",
                                              original_name=file_path.name,
                                              parents=file_parents)
                 file_resources.append(file_resource)

ted_sws/data_manager/adapters/notice_repository.py

@@ -1,18 +1,16 @@
-import copy
+import json
 import json
 import logging
 import pathlib
 from datetime import datetime
-from typing import Iterator, Union, Optional, Tuple, Any
+from typing import Iterator, Union, Optional, Any
 
-import gridfs
-from bson import ObjectId
 from pymongo import MongoClient, ASCENDING
 
 from ted_sws import config
 from ted_sws.core.model.lazy_object import LazyObjectFieldsLoaderABC
-from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation, Manifestation
-from ted_sws.core.model.metadata import NormalisedMetadata, TEDMetadata, Metadata, XMLMetadata
+from ted_sws.core.model.manifestation import XMLManifestation, RDFManifestation, METSManifestation
+from ted_sws.core.model.metadata import NormalisedMetadata, TEDMetadata, XMLMetadata
 from ted_sws.core.model.notice import Notice, NoticeStatus
 from ted_sws.data_manager.adapters import inject_date_string_fields, remove_date_string_fields
 from ted_sws.data_manager.adapters.manifestation_repository import XMLManifestationRepository, \

ted_sws/data_manager/services/mapping_suite_resource_manager.py

@@ -69,9 +69,11 @@ def mapping_suite_files_grouped_by_path(file_resources: List[FileResource], grou
     return grouped_files
 
 
-def read_flat_file_resources(path: pathlib.Path, file_resources=None, extension=None) -> List[FileResource]:
+def read_flat_file_resources(path: pathlib.Path, file_resources=None, extension=None, with_content=True) -> \
+        List[FileResource]:
     return MappingSuiteRepositoryInFileSystem.read_flat_file_resources(
         path=path,
         file_resources=file_resources,
-        extension=extension
+        extension=extension,
+        with_content=with_content
     )

ted_sws/mapping_suite_processor/adapters/conceptual_mapping_reader.py

@@ -13,7 +13,6 @@
     RULES_E_FORM_BT_NAME, RULES_FIELD_XPATH_CONDITION, CONCEPTUAL_MAPPINGS_REMARKS_SHEET_NAME, \
     CONCEPTUAL_MAPPINGS_CL2_ORGANISATIONS_SHEET_NAME, CONCEPTUAL_MAPPINGS_CL1_ROLES_SHEET_NAME, CL_MAPPING_REFERENCE, \
     CL_SUPERTYPE, CL_FIELD_VALUE, CL_XML_PATH_FRAGMENT
-from ted_sws.mapping_suite_processor.services.conceptual_mapping_files_injection import FILE_NAME_KEY
 from ted_sws.notice_validator import BASE_XPATH_FIELD
 
 # This set of constants refers to fields in the Conceptual Mapping file
@@ -31,6 +30,9 @@
 RULES_CLASS_PATH = 'Class path (M)'
 RULES_PROPERTY_PATH = 'Property path (M)'
 
+FILE_NAME_KEY = "File name"
+REF_INTEGRATION_TESTS_KEY = "Reference to Integration Tests (O)"
+
 
 class ConceptualMappingReader:
     """

ted_sws/mapping_suite_processor/resources/templates/conceptual_mappings_diff_report.jinja2

@@ -1,10 +1,20 @@
+{% set mapping_suite1 = metadata['mapping_suite_ids'][0] or metadata['mapping_suite_ids'][1] %}
+{% set mapping_suite2 = metadata['mapping_suite_ids'][1] or metadata['mapping_suite_ids'][0] %}
+
+{% set ms1_parts = mapping_suite1.split("_") %}
+{% set ms1_acronym = ms1_parts[ms1_parts|length - 1] if ms1_parts else "" %}
+{% set ms2_parts = mapping_suite2.split("_") %}
+{% set ms2_acronym = ms2_parts[ms2_parts|length - 1] if ms2_parts else "" %}
+
+{% set ms_acronyms_title = ms1_acronym + ("-" + ms2_acronym if ms2_acronym else "") %}
+
 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for Conceptual Mappings Diff</title>
+    <title>CM {% if ms_acronyms_title %}({{ ms_acronyms_title }}) {% endif %}Diff Report</title>
     <link rel="stylesheet" href="https://code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <style>
@@ -50,7 +60,7 @@
     </style>
   </head>
   <body>
-	<h1>Conceptual Mappings Diff HTML report </h1>
+	<h1>Conceptual Mappings Diff Report</h1>
 	<hr>
     <ul>
         <li>Created at: {{ created_at }}</li>
@@ -75,8 +85,6 @@
                 <td>{{ branch1 }}</td>
                 <td>{{ branch2 }}</td>
             </tr>
-            {% set mapping_suite1 = metadata['mapping_suite_ids'][0] or metadata['mapping_suite_ids'][1] %}
-            {% set mapping_suite2 = metadata['mapping_suite_ids'][1] or metadata['mapping_suite_ids'][0] %}
             <tr>
                 <th>Mapping Suite</th>
                 <td>{{ mapping_suite1 }}</td>

ted_sws/mapping_suite_processor/services/conceptual_mapping_files_injection.py

@@ -2,10 +2,10 @@
 import shutil
 
 import pandas as pd
-from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME, \
-    CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME
 
-FILE_NAME_KEY = "File name"
+from ted_sws.mapping_suite_processor import CONCEPTUAL_MAPPINGS_RESOURCES_SHEET_NAME, \
+    CONCEPTUAL_MAPPINGS_RML_MODULES_SHEET_NAME, CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME
+from ted_sws.mapping_suite_processor.adapters.conceptual_mapping_reader import FILE_NAME_KEY, REF_INTEGRATION_TESTS_KEY
 
 
 def mapping_suite_processor_inject_resources(conceptual_mappings_file_path: pathlib.Path,
@@ -83,3 +83,28 @@ def mapping_suite_processor_inject_sparql_queries(sparql_queries_folder_path: pa
     :return:
     """
     shutil.copytree(sparql_queries_folder_path, output_sparql_queries_folder_path, dirs_exist_ok=True)
+
+
+def mapping_suite_processor_inject_integration_sparql_queries(
+        conceptual_mappings_file_path: pathlib.Path,
+        sparql_queries_folder_path: pathlib.Path,
+        output_sparql_queries_folder_path: pathlib.Path
+):
+    """
+        This function reads the SPARQL files from conceptual_mappings_file_path Rules sheet, and then, based on this list,
+        the resources in sparql_queries_folder_path will be copied to output_sparql_queries_folder_path.
+        :param conceptual_mappings_file_path:
+        :param sparql_queries_folder_path:
+        :param output_sparql_queries_folder_path:
+        :return:
+        """
+    sparql_files_df = pd.read_excel(conceptual_mappings_file_path, sheet_name=CONCEPTUAL_MAPPINGS_RULES_SHEET_NAME,
+                                    skiprows=1)
+    sparql_df_values = sparql_files_df[REF_INTEGRATION_TESTS_KEY].dropna().str.split(',').values.tolist()
+    sparql_files_names = list(set([item.strip() for sublist in sparql_df_values for item in sublist]))
+    for sparql_file_name in sparql_files_names:
+        src_sparql_file_path = sparql_queries_folder_path / sparql_file_name
+        dest_sparql_file_path = output_sparql_queries_folder_path / sparql_file_name
+        if src_sparql_file_path.exists():
+            shutil.copy(src_sparql_file_path, dest_sparql_file_path)
+

ted_sws/notice_validator/resources/templates/_ms_meta_title_pre.jinja2

@@ -0,0 +1,12 @@
+{% set notice_in_title = "" %}
+
+{% if notice_ids %}
+    {% set notice_in_title = notice_ids[0] if notice_ids|length == 1 else "" %}
+{% elif notices %}
+    {% set notice_in_title = notices[0].notice_id if notices|length == 1 else "" %}
+{%  endif %}
+
+{% set ms_parts = mapping_suite_identifier.split("_") %}
+{% set ms_acronym = ms_parts[1] if ms_parts and ms_parts|length >= 2 else ms_parts[0] %}
+{% set report_meta_title = ms_acronym + ("/" + notice_in_title if notice_in_title else "") %}
+{% if report_meta_title %}{{ report_meta_title }} {% endif %}

ted_sws/notice_validator/resources/templates/shacl_shape_validation_results_report.jinja2

@@ -4,7 +4,7 @@
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for SHACL Validation</title>
+    <title>{% include '_ms_meta_title_pre.jinja2' %}SHACL Validation Report</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <style type="text/css">
         body {
@@ -56,26 +56,28 @@
     </style>
 </head>
 <body>
-<h1>SHACL Validation HTML report </h1>
+<h1>SHACL Validation Report</h1>
 <hr>
 <h2>Report details: </h2>
 <ul>
     <li>Date created: {{ created }}</li>
     <li>SHACL test suite identifier: {{ test_suite_identifier }}</li>
     <li>Mapping suite identifier: {{ mapping_suite_identifier }}</li>
     {% if notice_ids %}
-    <li><div><hr></div>Notice identifier(s):
-        <div data-role="collapsible" data-state="{% if notice_ids|length > 1 %}collapsed{% endif %}"
+    <li><div><hr></div>
+        {%  set nr_notices = notice_ids|length %}
+        {% if nr_notices > 1 %}
+        Notice identifiers ({{ nr_notices }}):
+        <div data-role="collapsible" data-state="collapsed"
             class="collapsible-wrapper">
             <h4><a href="#"></a></h4>
             <div class="ui-collapsible-content">
-            {% if notice_ids|length == 1 %}
-                {{ notice_ids[0] }}
-            {% else %}
                 {{ notice_ids | tojson(indent=2) | replace('\n', '<br>') }}
-            {% endif %}
             </div>
         </div>
+        {% elif nr_notices == 1 %}
+            <b>Notice identifier: {{ notice_ids[0] }}</b>
+        {% endif %}
     </li>
     {% endif %}
 </ul>

ted_sws/notice_validator/resources/templates/sparql_query_results_report.jinja2

@@ -1,14 +1,15 @@
 {% macro increment(dictionary, key, increment_by=1) %}
     {% if dictionary.update({key: dictionary[key] + increment_by}) %} {% endif %}
 {% endmacro %}
+
 {% set results_count = validation_results|length %}
 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for SPARQL Validation</title>
+    <title>{% include '_ms_meta_title_pre.jinja2' %}SPARQL Validation Report</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <link rel="stylesheet" href="https://code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css">
     <style type="text/css">
@@ -87,26 +88,28 @@
     </style>
   </head>
   <body>
-	<h1>SPARQL Validation HTML report </h1>
+	<h1>SPARQL Validation Report</h1>
 	<hr>
     <h2>Report details: </h2>
     <ul>
         <li>Date created: {{ created }}</li>
         <li>SPARQL test suite identifier: {{ test_suite_identifier }}</li>
         <li>Mapping suite identifier: {{ mapping_suite_identifier }}</li>
         {% if notice_ids %}
-        <li><div><hr></div>Notice identifier(s):
-            <div data-role="collapsible" data-state="{% if notice_ids|length > 1 %}collapsed{% endif %}"
+        <li><div><hr></div>
+            {%  set nr_notices = notice_ids|length %}
+            {% if nr_notices > 1 %}
+            Notice identifiers ({{ nr_notices }}):
+            <div data-role="collapsible" data-state="collapsed"
                 class="collapsible-wrapper">
                 <h4><a href="#"></a></h4>
                 <div class="ui-collapsible-content">
-                {% if notice_ids|length == 1 %}
-                    {{ notice_ids[0] }}
-                {% else %}
                     {{ notice_ids | tojson(indent=2) | replace('\n', '<br>') }}
-                {% endif %}
                 </div>
             </div>
+            {% elif nr_notices == 1 %}
+                <b>Notice identifier: {{ notice_ids[0] }}</b>
+            {% endif %}
         </li>
         {% endif %}
     </ul>

ted_sws/notice_validator/resources/templates/validation_summary_report.jinja2

@@ -1,10 +1,12 @@
+{% set mapping_suite_identifier = xml_manifestation.xpath_coverage_summary.mapping_suite_identifier %}
+
 <!DOCTYPE html>
 <html lang="en">
   <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for SPARQL Validation</title>
+    <title>{% include '_ms_meta_title_pre.jinja2' %}Summary Validation Report</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
     <style type="text/css">
         body {
@@ -68,29 +70,35 @@
     </style>
   </head>
   <body>
-	<h1>Validation Summary HTML report </h1>
+	<h1>Validation Summary Report</h1>
 	<hr>
     <h2>Report details: </h2>
     <ul>
         <li>Date created: {{ created }}</li>
-        <li><div><hr></div>Notice identifier(s):
-            <div data-role="collapsible" data-state="{% if notices|length > 1 %}collapsed{% endif %}"
+        <li><b>Mapping suite identifier:</b> {{ mapping_suite_identifier }}</li>
+        {%  if template_metadata.grouping %}
+            <li><b>Grouping:</b> {{ template_metadata.grouping }}</li>
+        {%  endif %}
+        <li><div><hr></div>
+            {%  set nr_notices = notices|length %}
+            {% if nr_notices > 1 %}
+            <b>Notice identifiers ({{ nr_notices }}):</b>
+            <div data-role="collapsible" data-state="collapsed"
                 class="collapsible-wrapper">
                 <h4><a href="#"></a></h4>
                 <div class="ui-collapsible-content">
-                {% if notices|length == 1 %}
-                    {{ notices[0].notice_id }}
-                {% else %}
                     <ul>
                     {% for report_data in notices %}
                         <li>
                             <a href="file://{{ template_metadata.package_output_path }}/{{ report_data.path }}/{{ report_data.notice_id }}" target="_blank">{{ report_data.notice_id }}</a>
                         </li>
                     {% endfor %}
                     </ul>
-                {% endif %}
                 </div>
             </div>
+            {% elif nr_notices == 1 %}
+                <b>Notice identifier: {{ notices[0].notice_id }}</b>
+            {% endif %}
         </li>
     </ul>
     <hr>

ted_sws/notice_validator/resources/templates/xpath_coverage_report.jinja2

@@ -1,10 +1,11 @@
+{% set notices = validation_result.notices %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta http-equiv="X-UA-Compatible" content="ie=edge">
-    <title>HTML report for XPATH Coverage</title>
+    <title>{% include '_ms_meta_title_pre.jinja2' %}XPATH Coverage Report</title>
     <link href="https://cdn.datatables.net/1.10.22/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
 
     <style type="text/css">
@@ -71,7 +72,7 @@
     </style>
 </head>
 <body>
-<h1>XPATH Coverage HTML report </h1>
+<h1>XPATH Coverage Report</h1>
 <hr>
 <h2>Report details: </h2>
 <ul>
@@ -90,23 +91,23 @@
         <h3>{{ "%.2f"|format((validation_result.conceptual_coverage|float * 100)) }}%</h3>
         <p>(Number of Conceptual XPATHs found in notices) / (Total number of unique Conceptual XPATHs)</p>
     </li>
-    {%  set nr_notices = validation_result.notices|length %}
-    <li><div><hr></div>Notice identifier(s):
-        <div data-role="collapsible" data-state="{% if nr_notices > 1 %}collapsed{% endif %}"
-            class="collapsible-wrapper">
+    {%  set nr_notices = notices|length %}
+    <li><div><hr></div>
+        {% if nr_notices > 1 %}
+            <b>Notice identifiers ({{ nr_notices }}):</b>
+        <div data-role="collapsible" data-state="collapsed" class="collapsible-wrapper">
             <h4><a href="#"></a></h4>
             <div class="ui-collapsible-content">
-            {% if nr_notices == 1 %}
-                {{ validation_result.notices[0].notice_id }}
-            {% else %}
                 <ul>
-                {% for report_data in validation_result.notices %}
+                {% for report_data in notices %}
                     <li><a href="file://{{ template_metadata.package_output_path }}/{{ report_data.path }}/{{ report_data.notice_id }}/test_suite_report/xpath_coverage_validation.html" target="_blank">{{ report_data.notice_id }}</a></li>
                 {% endfor %}
                 </ul>
-            {% endif %}
             </div>
         </div>
+        {% elif nr_notices == 1 %}
+            <b>Notice identifier: {{ notices[0].notice_id }}</b>
+        {% endif %}
     </li>
 </ul>
 {% if validation_result.xpath_assertions|length > 0 %}