Add image-registry configuration reading (hitachienergy#3106)

ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py

@@ -231,7 +231,9 @@ def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: D
         for feature in manifest['detected-features']:
             lines.append(f'- {feature}')
 
-        for reqs in [('files', 'Files'), ('grafana-dashboards', 'Dashboards')]:
+        for reqs in [('files', 'Files'),
+                     ('grafana-dashboards', 'Dashboards'),
+                     ('images', 'Images')]:
             reqs_to_download = sorted(requirements[reqs[0]])
             if reqs_to_download:
                 lines.append('')
@@ -245,6 +247,10 @@ def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: D
 
     def __filter_manifest(self, requirements: Dict[str, Any], manifest: Dict[str, Any]):
         """
+        Filter entries in the `requirements` based on the parsed `manifest` documents.
+
+        :param requirements: parsed requirements which will be filtered based on the `manifest` output
+        :param manifest: parsed documents which will be used to filter `requirements`
         """
         if 'grafana' not in manifest['detected-features']:
             requirements['grafana-dashboards'] = []
@@ -259,6 +265,16 @@ def __filter_manifest(self, requirements: Dict[str, Any], manifest: Dict[str, An
         if files_to_exclude:
             requirements['files'] = {url: data for url, data in files.items() if url not in files_to_exclude}
 
+        if len(manifest['detected-images']):
+            images = requirements['images']
+            images_to_exclude: List[str] = []
+            for image in images:
+                if image not in manifest['detected-images']:
+                    images_to_exclude.append(image)
+
+            if images_to_exclude:
+                requirements['images'] = {name: data for name, data in images.items() if name not in images_to_exclude}
+
     def read_manifest(self, requirements: Dict[str, Any]):
         """
         Construct ManifestReader and parse only required data.
@@ -269,7 +285,7 @@ def read_manifest(self, requirements: Dict[str, Any]):
         if not self.dest_manifest:
             return
 
-        mreader = ManifestReader(self.dest_manifest)
+        mreader = ManifestReader(self.dest_manifest, self.os_arch)
         manifest = mreader.parse_manifest()
         self.__filter_manifest(requirements, manifest)
 

ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py

@@ -3,6 +3,7 @@
 
 import yaml
 
+from src.config.os_type import OSArch
 from src.error import CriticalError
 
 
@@ -26,10 +27,12 @@ class ManifestReader:
     Main running method is :func:`~manifest_reader.ManifestReader.parse_manifest` which returns formatted manifest output.
     """
 
-    def __init__(self, dest_manifest: Path):
+    def __init__(self, dest_manifest: Path, arch: OSArch):
         self.__dest_manifest = dest_manifest
+        self.__os_arch: str = arch.value
         self.__detected_components: Set = set()
         self.__detected_features: Set = set()
+        self.__detected_images: Set = set()
 
     def __parse_cluster_info(self, cluster_doc: Dict):
         """
@@ -53,13 +56,28 @@ def __parse_feature_mappings_info(self, feature_mappings_doc: Dict):
             for feature in mappings[mapping]:
                 self.__detected_features.add(feature)
 
+    def __parse_image_registry_info(self, image_registry_doc: Dict):
+        """
+        Parse `configuration/image-registry` document and extract only used images.
+
+        :param image_registry_doc: handler to a `configuration/image-registry` document
+        """
+        self.__detected_images.add(image_registry_doc['specification']['registry_image']['name'])
+
+        target_arch_images = image_registry_doc['specification']['images_to_load'][self.__os_arch]
+        for target_images in target_arch_images:
+            for image in target_arch_images[target_images]:
+                self.__detected_images.add(image['name'])
+
     def parse_manifest(self) -> Dict[str, Any]:
         """
         Load the manifest file, call parsers on required docs and return formatted output.
         """
+        required_docs: Set[str] = {'epiphany-cluster', 'configuration/feature-mappings'}
         parse_doc: Dict[str, Callable] = {
             'epiphany-cluster':               self.__parse_cluster_info,
-            'configuration/feature-mappings': self.__parse_feature_mappings_info
+            'configuration/feature-mappings': self.__parse_feature_mappings_info,
+            'configuration/image-registry':   self.__parse_image_registry_info
         }
 
         parsed_docs: Set[str] = set()
@@ -71,8 +89,9 @@ def parse_manifest(self) -> Dict[str, Any]:
             except KeyError:
                 pass
 
-        if len(parsed_docs) != len(parse_doc.keys()):
-            raise CriticalError(f'ManifestReader - could not find documents: {parsed_docs ^ parse_doc.keys()}')
+        if len(parsed_docs) < len(required_docs):
+            raise CriticalError(f'ManifestReader - could not find documents: {parsed_docs ^ required_docs}')
 
         return {'detected-components': sorted(list(self.__detected_components)),
-                'detected-features': sorted(list(self.__detected_features))}
+                'detected-features': sorted(list(self.__detected_features)),
+                'detected-images': sorted(list(self.__detected_images))}

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py

@@ -9,59 +9,57 @@
                                EXPECTED_VERBOSE_OUTPUT,
                                EXPECTED_VERBOSE_DASHBOARD_OUTPUT,
                                EXPECTED_VERBOSE_FILE_OUTPUT,
-                               FILE_REQUIREMENTS)
+                               EXPECTED_VERBOSE_IMAGE_OUTPUT,
+                               FILE_REQUIREMENTS,
+                               IMAGE_REQUIREMENTS)
 from tests.data.manifest_reader import (INPUT_MANIFEST_FEATURE_MAPPINGS,
-                                        INPUT_MANIFEST_WITH_DASHBOARDS)
+                                        INPUT_MANIFEST_WITH_DASHBOARDS,
+                                        INPUT_MANIFEST_WITH_IMAGES)
+from src.config.os_type import OSArch
 
 
-@pytest.fixture(scope='class')
-def default_manifest(request):
+@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, REQUIREMENTS',
+                         [
+                          (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_FILE_OUTPUT, FILE_REQUIREMENTS),
+                          (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT, DASHBOARD_REQUIREMENTS),
+                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS),
+                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_VERBOSE_IMAGE_OUTPUT, IMAGE_REQUIREMENTS)
+                         ])
+def test_manifest_verbose_output(INPUT_DOC: str,
+                                 EXPECTED_OUTPUT_DOC: str,
+                                 REQUIREMENTS: str,
+                                 mocker, caplog):
     """
-    Generate default requirements for each test
+    Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest`
+
+    :param INPUT_DOC: yaml doc which will be parsed by the ManifestReader
+    :param EXPECTED_OUTPUT_DOC: expected output to be printed by the `Config` class, then tested against the parsed `INPUT_DOC`
+    :param REQUIREMENTS: yaml doc containing requirements passed to `Config`'s read_manifest()
     """
-    request.cls.MANIFEST = {
+    MANIFEST = {
         'files': '',
-        'grafana-dashboards': ''
+        'grafana-dashboards': '',
+        'images': ''
     }
 
+    mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
+    caplog.set_level(logging.INFO)
 
-@pytest.mark.usefixtures('default_manifest')
-class TestConfig:
-    @pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, REQUIREMENTS',
-                             [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT, DASHBOARD_REQUIREMENTS),
-                              (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS),
-                              (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_FILE_OUTPUT, FILE_REQUIREMENTS)
-                             ])
-    def test_manifest_verbose_output(self, INPUT_DOC: str,
-                                     EXPECTED_OUTPUT_DOC: str,
-                                     REQUIREMENTS: str,
-                                     mocker, caplog):
-        """
-        Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest`
-
-        :param INPUT_DOC: yaml doc which will be parsed by the ManifestReader
-        :param EXPECTED_OUTPUT_DOC: expected output to be printed by the `Config` class, then tested against the parsed `INPUT_DOC`
-        :param REQUIREMENTS: yaml doc containing requirements passed to `Config`'s read_manifest()
-        """
-
-        mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
-        caplog.set_level(logging.INFO)
-
-        # mock Config's init methods:
-        Config._Config__add_args = lambda *args: None
-        Config._Config__log_info_summary = lambda *args: None
-
-        config = Config([])
+    # mock Config's init methods:
+    Config._Config__add_args = lambda *args: None
+    Config._Config__log_info_summary = lambda *args: None
 
-        # mock required config data:
-        config.dest_manifest = Path('/some/path')
-        config.verbose_mode = True
+    config = Config([])
 
-        req_key, doc = tuple(yaml.safe_load(REQUIREMENTS).items())[0]
+    # mock required config data:
+    config.dest_manifest = Path('/some/path')
+    config.os_arch = OSArch.X86_64
+    config.verbose_mode = True
 
-        self.MANIFEST[req_key] = doc
-        config.read_manifest(self.MANIFEST)
+    req_key, doc = tuple(yaml.safe_load(REQUIREMENTS).items())[0]
+    MANIFEST[req_key] = doc
+    config.read_manifest(MANIFEST)
 
-        log_output = f'\n{"".join(caplog.messages)}\n'
+    log_output = f'\n{"".join(caplog.messages)}\n'
 
-        assert log_output == EXPECTED_OUTPUT_DOC
+    assert log_output == EXPECTED_OUTPUT_DOC

ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py

@@ -4,17 +4,23 @@
 import yaml
 
 from src.config.manifest_reader import ManifestReader
+from src.config.os_type import OSArch
 from tests.data.manifest_reader import (EXPECTED_FEATURE_MAPPINGS,
                                         EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS,
+                                        EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64,
+                                        EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64,
                                         INPUT_MANIFEST_FEATURE_MAPPINGS,
-                                        INPUT_MANIFEST_WITH_DASHBOARDS)
+                                        INPUT_MANIFEST_WITH_DASHBOARDS,
+                                        INPUT_MANIFEST_WITH_IMAGES)
 
-@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC',
-                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS),
-                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS)])
-def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker):
+@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH',
+                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS, OSArch.X86_64),
+                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS, OSArch.X86_64),
+                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64, OSArch.X86_64),
+                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64, OSArch.ARM64)])
+def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH, mocker):
     ''' Check manifest file parsing '''
     mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
 
-    mreader = ManifestReader(Path('/some/path'))
+    mreader = ManifestReader(Path('/some/path'), OS_ARCH)
     assert mreader.parse_manifest() == EXPECTED_OUTPUT_DOC