Add image-registry configuration reading (hitachienergy#3106)

sbbroot · May 20, 2022 · a5f1656 · a5f1656
1 parent fa27451
commit a5f1656
Show file tree

Hide file tree

Showing 6 changed files with 429 additions and 14 deletions.
diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py
@@ -252,7 +252,7 @@ def read_manifest(self, requirements: Dict[str, Any]):
         if not self.dest_manifest:
             return
 
-        mreader = ManifestReader(self.dest_manifest)
+        mreader = ManifestReader(self.dest_manifest, self.os_arch)
         output = mreader.parse_manifest()
 
         if 'grafana' not in output['detected-features']:

diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py
@@ -3,6 +3,7 @@
 
 import yaml
 
+from src.config.os_type import OSArch
 from src.error import CriticalError
 
 
@@ -26,10 +27,12 @@ class ManifestReader:
     Main running method is :func:`~manifest_reader.ManifestReader.parse_manifest` which returns formatted manifest output.
     """
 
-    def __init__(self, dest_manifest: Path):
+    def __init__(self, dest_manifest: Path, arch: OSArch):
         self.__dest_manifest = dest_manifest
+        self.__os_arch: str = arch.value
         self.__detected_components: Set = set()
         self.__detected_features: Set = set()
+        self.__detected_images: Set = set()
 
     def __parse_cluster_info(self, cluster_doc: Dict):
         """
@@ -53,13 +56,28 @@ def __parse_feature_mappings_info(self, feature_mappings_doc: Dict):
             for feature in mappings[mapping]:
                 self.__detected_features.add(feature)
 
+    def __parse_image_registry_info(self, image_registry_doc: Dict):
+        """
+        Parse `configuration/image-registry` document and extract only used images.
+
+        :param image_registry_doc: handler to a `configuration/image-registry` document
+        """
+        self.__detected_images.add(image_registry_doc['specification']['registry_image']['name'])
+
+        target_arch_images = image_registry_doc['specification']['images_to_load'][self.__os_arch]
+        for target_images in target_arch_images:
+            for image in target_arch_images[target_images]:
+                self.__detected_images.add(image['name'])
+
     def parse_manifest(self) -> Dict[str, Any]:
         """
         Load the manifest file, call parsers on required docs and return formatted output.
         """
+        required_docs: Set[str] = {'epiphany-cluster', 'configuration/feature-mappings'}
         parse_doc: Dict[str, Callable] = {
             'epiphany-cluster':               self.__parse_cluster_info,
-            'configuration/feature-mappings': self.__parse_feature_mappings_info
+            'configuration/feature-mappings': self.__parse_feature_mappings_info,
+            'configuration/image-registry':   self.__parse_image_registry_info
         }
 
         parsed_docs: Set[str] = set()
@@ -71,8 +89,9 @@ def parse_manifest(self) -> Dict[str, Any]:
             except KeyError:
                 pass
 
-        if len(parsed_docs) != len(parse_doc.keys()):
-            raise CriticalError(f'ManifestReader - could not find documents: {parsed_docs ^ parse_doc.keys()}')
+        if len(parsed_docs) < len(required_docs):
+            raise CriticalError(f'ManifestReader - could not find documents: {parsed_docs ^ required_docs}')
 
         return {'detected-components': sorted(list(self.__detected_components)),
-                'detected-features': sorted(list(self.__detected_features))}
+                'detected-features': sorted(list(self.__detected_features)),
+                'detected-images': sorted(list(self.__detected_images))}
diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py
@@ -5,6 +5,7 @@
 import yaml
 
 from src.config.config import Config
+from src.config.os_type import OSArch
 from tests.data.config import EXPECTED_VERBOSE_OUTPUT, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS
 from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS, INPUT_MANIFEST_WITH_DASHBOARDS
 
@@ -26,6 +27,7 @@ def test_manifest_verbose_output(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker, caplog)
 
     # mock required config data:
     config.dest_manifest = Path('/some/path')
+    config.os_arch = OSArch.X86_64
     config.verbose_mode = True
     config.read_manifest(yaml.safe_load(DASHBOARD_REQUIREMENTS))
 

diff --git a/...aybooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py b/...aybooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py
@@ -4,17 +4,23 @@
 import yaml
 
 from src.config.manifest_reader import ManifestReader
+from src.config.os_type import OSArch
 from tests.data.manifest_reader import (EXPECTED_FEATURE_MAPPINGS,
                                         EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS,
+                                        EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64,
+                                        EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64,
                                         INPUT_MANIFEST_FEATURE_MAPPINGS,
-                                        INPUT_MANIFEST_WITH_DASHBOARDS)
+                                        INPUT_MANIFEST_WITH_DASHBOARDS,
+                                        INPUT_MANIFEST_WITH_IMAGES)
 
-@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC',
-                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS),
-                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS)])
-def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker):
+@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH',
+                         [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS, OSArch.X86_64),
+                          (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS, OSArch.X86_64),
+                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64, OSArch.X86_64),
+                          (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64, OSArch.ARM64)])
+def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH, mocker):
     ''' Check manifest file parsing '''
     mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC))
 
-    mreader = ManifestReader(Path('/some/path'))
+    mreader = ManifestReader(Path('/some/path'), OS_ARCH)
     assert mreader.parse_manifest() == EXPECTED_OUTPUT_DOC
diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py
@@ -41,6 +41,199 @@
     sha256: 66340fa3256d432287cba75ab5177eb058c77afa7d521a75d58099f95b1bff50
 """
 
+IMAGES_REQUIREMENTS = """
+images:
+  'haproxy:2.2.2-alpine':
+    sha1: dff8993b065b7f7846adb553548bcdcfcd1b6e8e
+
+  'kubernetesui/dashboard:v2.3.1':
+    sha1: 8c8a4ac7a643f9c5dd9e5d22876c434187312db8
+
+  'kubernetesui/metrics-scraper:v1.0.7':
+    sha1: 5a0052e2afd3eef3ae638be21938b29b1d608ebe
+
+  'registry:2.8.0':
+    sha1: 89795c17099199c752d02ad8797c1d4565a08aff
+    allow_mismatch: true
+
+  # applications
+  'bitnami/pgpool:4.2.4':
+    sha1: 66741f3cf4a508bd1f80e2965b0086a4c0fc3580
+
+  'bitnami/pgbouncer:1.16.0':
+    sha1: f2e37eecbf9aed44d5566f06dcc101c1ba9edff9
+
+  'epiphanyplatform/keycloak:14.0.0':
+    sha1: b59d75a967cedd3a4cf5867eced2fb5dff52f60e
+
+  'rabbitmq:3.8.9':
+    sha1: c64408bf5bb522f47d5323652dd5e60560dcb5bc
+
+  # K8s
+  # v1.18.6
+  'k8s.gcr.io/kube-apiserver:v1.18.6':
+    sha1: 164968226f4617abaa31e6108ed9034a1e302f4f
+
+  'k8s.gcr.io/kube-controller-manager:v1.18.6':
+    sha1: ebea3fecab9e5693d31438fa37dc4d02c6914d67
+
+  'k8s.gcr.io/kube-scheduler:v1.18.6':
+    sha1: 183d29c4fdcfda7478d08240934fdb6845e2e3ec
+
+  'k8s.gcr.io/kube-proxy:v1.18.6':
+    sha1: 62da886e36efff0c03a16e19c1442a1c3040fbf1
+
+  'k8s.gcr.io/coredns:1.6.7':
+    sha1: 76615ffabb22fd4fb3d562cb6ebcd243f8826e48
+
+  'k8s.gcr.io/etcd:3.4.3-0':
+    sha1: 6ee82ddb1bbc7f1831c42046612b8bcfbb171b45
+
+  'quay.io/coreos/flannel:v0.12.0-amd64':
+    sha1: 3516522e779373983992095e61eb6615edd50d1f
+
+  'quay.io/coreos/flannel:v0.12.0':
+    sha1: 2cb6ce8f1361886225526767c4a0422c039453c8
+
+  'calico/cni:v3.15.0':
+    sha1: aa59f624c223bc398a42c7ba9e628e8143718e58
+
+  'calico/kube-controllers:v3.15.0':
+    sha1: f8921f5d67ee7db1c619aa9fdb74114569684ceb
+
+  'calico/node:v3.15.0':
+    sha1: b15308e1aa8b9c56253c142e4361e47125bb4ac5
+
+  'calico/pod2daemon-flexvol:v3.15.0':
+    sha1: dd1a6525bde05937a28e3d9176b826162ae489af
+
+  # v1.19.15
+  'k8s.gcr.io/kube-apiserver:v1.19.15':
+    sha1: e01c8d778e4e693a0ea09cdbbe041a65cf070c6f
+
+  'k8s.gcr.io/kube-controller-manager:v1.19.15':
+    sha1: d1f5cc6a861b2259861fb78b2b83e9a07b788e31
+
+  'k8s.gcr.io/kube-scheduler:v1.19.15':
+    sha1: b07fdd17205bc071ab108851d245689642244f92
+
+  'k8s.gcr.io/kube-proxy:v1.19.15':
+    sha1: 9e2e7a8d40840bbade3a1f2dc743b9226491b6c2
+
+  # v1.20.12
+  'k8s.gcr.io/kube-apiserver:v1.20.12':
+    sha1: bbb037b9452db326aaf09988cee080940f3c418a
+
+  'k8s.gcr.io/kube-controller-manager:v1.20.12':
+    sha1: 4a902578a0c548edec93e0f4afea8b601fa54b93
+
+  'k8s.gcr.io/kube-scheduler:v1.20.12':
+    sha1: ed5ceb21d0f5bc350db69550fb7feac7a6f1e50b
+
+  'k8s.gcr.io/kube-proxy:v1.20.12':
+    sha1: f937aba709f52be88360361230840e7bca756b2e
+
+  'k8s.gcr.io/coredns:1.7.0':
+    sha1: 5aa15f4cb942885879955b98a0a824833d9f66eb
+
+  'k8s.gcr.io/pause:3.2':
+    sha1: ae4799e1a1ec9cd0dda8ab643b6e50c9fe505fef
+
+  # v1.21.7
+  'k8s.gcr.io/kube-apiserver:v1.21.7':
+    sha1: edb26859b3485808716982deccd90ca420828649
+
+  'k8s.gcr.io/kube-controller-manager:v1.21.7':
+    sha1: 9abf1841da5b113b377c1471880198259ec2d246
+
+  'k8s.gcr.io/kube-scheduler:v1.21.7':
+    sha1: 996d25351afc96a10e9008c04418db07a99c76b7
+
+  'k8s.gcr.io/kube-proxy:v1.21.7':
+    sha1: 450af22a892ffef276d4d58332b7817a1dde34e7
+
+  'k8s.gcr.io/coredns/coredns:v1.8.0':
+    sha1: 03114a98137e7cc2dcf4983b919e6b93ac8d1189
+
+  'k8s.gcr.io/etcd:3.4.13-0':
+    sha1: d37a2efafcc4aa86e6dc497e87e80b5d7f326115
+
+  'k8s.gcr.io/pause:3.4.1':
+    sha1: 7f57ae28d733f99c0aab8f4e27d4b0c034cd0c04
+
+  # v1.22.4
+  'k8s.gcr.io/kube-apiserver:v1.22.4':
+    sha1: 2bf4ddb2e1f1530cf55ebaf8e8d0c56ad378b9ec
+
+  'k8s.gcr.io/kube-controller-manager:v1.22.4':
+    sha1: 241924fa3dc4671fe6644402f7beb60028c02c71
+
+  'k8s.gcr.io/kube-scheduler:v1.22.4':
+    sha1: 373e2939072b03cf5b1e115820b7fb6b749b0ebb
+
+  'k8s.gcr.io/kube-proxy:v1.22.4':
+    sha1: fecfb88509a430c29267a99b83f60f4a7c333583
+
+  'k8s.gcr.io/coredns/coredns:v1.8.4':
+    sha1: 69c8e14ac3941fd5551ff22180be5f4ea2742d7f
+
+  'k8s.gcr.io/etcd:3.5.0-0':
+    sha1: 9d9ee2df54a201dcc9c7a10ea763b9a5dce875f1
+
+  'k8s.gcr.io/pause:3.5':
+    sha1: bf3e3420df62f093f94c41d2b7a62b874dcbfc28
+
+  'quay.io/coreos/flannel:v0.14.0-amd64':
+    sha1: cff47465996a51de4632b53abf1fca873f147027
+
+  'quay.io/coreos/flannel:v0.14.0':
+    sha1: a487a36f7b31677e50e74b96b944f27fbce5ac13
+
+  'calico/cni:v3.20.3':
+    sha1: 95e4cf79e92715b13e500a0efcfdb65590de1e04
+
+  'calico/kube-controllers:v3.20.3':
+    sha1: 5769bae60830abcb3c5d97eb86b8f9938a587b2d
+
+  'calico/node:v3.20.3':
+    sha1: cc3c8727ad30b4850e8d0042681342a4f2351eff
+
+  'calico/pod2daemon-flexvol:v3.20.3':
+    sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e
+
+  # --- Rook ---
+  'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0':
+    sha1: f076bd75359c6449b965c48eb8bad96c6d40790d
+
+  'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0':
+    sha1: 129eb73c8e118e5049fee3d273b2d477c547e080
+
+  'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0':
+    sha1: 2b45e5a3432cb89f3aec59584c1fa92c069e7a38
+
+  'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0':
+    sha1: ce5c57454254c195762c1f58e1d902d7e81ea669
+
+  'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1':
+    sha1: be1cf43617eea007629c0eb99149a99b6498f889
+
+  'quay.io/ceph/ceph:v16.2.7':
+    sha1: fe9b7802c67e19111f83ffe4754ab62df66fd417
+    allow_mismatch: true
+
+  'quay.io/cephcsi/cephcsi:v3.5.1':
+    sha1: 51dee9ea8ad76fb95ebd16f951e8ffaaaba95eb6
+
+  'quay.io/csiaddons/k8s-sidecar:v0.2.1':
+    sha1: f0fd757436ac5075910c460c1991ff67c4774d09
+
+  'quay.io/csiaddons/volumereplication-operator:v0.3.0':
+    sha1: d3cd17f14fcbf09fc6c8c2c5c0419f098f87a70f
+
+  'rook/ceph:v1.8.8':
+    sha1: f34039b17b18f5a855b096d48ff787b4013615e4
+"""
+
 
 EXPECTED_VERBOSE_OUTPUT = """
 Manifest summary:
@@ -95,3 +288,36 @@
 - grafana_dashboard_10991
 --------------------------------------------------
 """
+
+
+EXPECTED_VERBOSE_IMAGE_OUTPUT = """
+Manifest summary:
+--------------------------------------------------
+Components detected:
+- monitoring
+- repository
+
+Features detected:
+- filebeat
+- firewall
+- grafana
+- image-registry
+- node-exporter
+- prometheus
+- repository
+
+Images to download:
+- 'epiphanyplatform/keycloak:14.0.0',
+- 'haproxy:2.2.2-alpine',
+- 'k8s.gcr.io/coredns/coredns:v1.8.0',
+- 'k8s.gcr.io/etcd:3.4.13-0',
+- 'k8s.gcr.io/kube-apiserver:v1.21.7',
+- 'k8s.gcr.io/kube-apiserver:v1.22.4',
+- 'k8s.gcr.io/kube-scheduler:v1.21.7',
+- 'k8s.gcr.io/pause:3.4.1',
+- 'kubernetesui/dashboard:v2.3.1',
+- 'kubernetesui/metrics-scraper:v1.0.7',
+- 'rabbitmq:3.8.9',
+- 'registry:2.8.0'
+--------------------------------------------------
+"""