Verify sha256/sha1 of downloaded files (#3014) (#3036)

* Verify sha256/sha1 of downloaded files (#3014)

ansible/playbooks/roles/image_registry/tasks/main.yml

@@ -36,15 +36,44 @@
       when:
         - check_docker.rc == 0
 
+    - name: Check if previous version registry is running and stop it if needed
+      become: true
+      block:
+        - name: Get current running registry image name
+          command: "{% raw %}docker ps --format '{{ .Image }}' -f 'name=registry' -f 'status=running'{% endraw %}"
+          register: running_registry_image
+
+        - name: Get current running registry container names
+          command: "{% raw %}docker ps --format '{{ .Names }}' -f 'name=registry' -f 'status=running'{% endraw %}"
+          register: running_registry_containers
+          when:
+            - running_registry_image.stdout
+            - specification.registry_image.name != running_registry_image.stdout
+
+        - set_fact:
+            container_name: "{{ running_registry_containers.stdout }}"
+          when: running_registry_containers.changed
+
+        - block:
+            - name: Stop previous version running registry container
+              command: docker stop '{{ container_name }}'
+
+            - name: Kill previous version running registry container
+              command: docker rm '{{ container_name }}'
+          when:
+            - container_name is defined
+            - container_name | length > 0
+      when: is_upgrade_run
+
     - name: Load registry image
       include_tasks: load-image.yml
       vars:
         docker_image: "{{ specification.registry_image }}"
 
     - name: Check if registry is running
       become: true
-      command: docker ps -q -f 'ancestor=registry:2' -f 'status=running'
-      register: regitry_up_check
+      command: docker ps -q -f 'ancestor={{ specification.registry_image.name }}' -f 'status=running'
+      register: current_registry_up_check
       changed_when: false
 
     # todo run registry with SSL - generate/copy certs, mount it to registry container
@@ -53,7 +82,7 @@
       command: >-
         docker run -d -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -p 5000:5000 --restart=always
         --name epiphany-registry {{ specification.registry_image.name }}
-      when: regitry_up_check.stdout | length == 0
+      when: current_registry_up_check.stdout | length == 0
 
     - name: Set images to load
       set_fact:

ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py

@@ -7,8 +7,8 @@
 import sys
 
 from src.command.toolchain import TOOLCHAINS
-from src.config import Config, OSType
-from src.error import CriticalError
+from src.config import Config
+from src.error import DownloadRequirementsError
 
 
 def install_missing_modules(config: Config):
@@ -82,7 +82,7 @@ def main(argv: List[str]) -> int:
 
         time_end = datetime.datetime.now() - time_begin
         logging.info(f'Total execution time: {str(time_end).split(".")[0]}')
-    except CriticalError:
+    except DownloadRequirementsError:
         return 1
 
     return 0

ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml

@@ -9,8 +9,9 @@ images:
   'kubernetesui/metrics-scraper:v1.0.7':
     sha1: 5a0052e2afd3eef3ae638be21938b29b1d608ebe
 
-  'registry:2':
-    sha1: 6afd2766e6880c62cb80d1ea85dbcc20d6ef16a4
+  'registry:2.8.0':
+    sha1: 89795c17099199c752d02ad8797c1d4565a08aff
+    allow_mismatch: true
 
   # applications
   'bitnami/pgpool:4.2.4':

ansible/playbooks/roles/repository/files/download-requirements/src/command/command.py

@@ -57,7 +57,7 @@ def run(self, args: List[str],
             if process.returncode == 0:
                 return process
 
-            logging.warn(process.stderr)
+            logging.warning(process.stderr)
 
         raise CriticalError(f'Retries count reached maximum, command: `{self.__command}`')
 
@@ -86,7 +86,7 @@ def __or__(self, command) -> str:
             if rproc.returncode == 0:
                 return output
 
-            logging.warn(lproc.stderr if not lproc.returncode == 0 else rproc.stderr)
+            logging.warning(lproc.stderr if not lproc.returncode == 0 else rproc.stderr)
 
         raise CriticalError(f'Retries count reached maximum, command: `{self.__command}`')
 

ansible/playbooks/roles/repository/files/download-requirements/src/config.py

@@ -158,7 +158,7 @@ def __setup_logger(self, log_level: str, log_file: Path, no_logfile: bool):
         log_levels = {
             # map input log level to Python's logging library
             'error': logging.ERROR,
-            'warn': logging.WARN,
+            'warn': logging.WARNING,
             'info': logging.INFO,
             'debug': logging.DEBUG
         }

ansible/playbooks/roles/repository/files/download-requirements/src/error.py

@@ -1,21 +1,32 @@
 import logging
 
 
-class CriticalError(Exception):
+class DownloadRequirementsError(Exception):
     """
-    Raised when there was an error that could not be fixed by
-    download-requirements script.
+    Base class for all non standard errors raised during a script run.
     """
-
     def __init__(self, msg: str):
         super().__init__()
         logging.error(msg)
 
 
+class CriticalError(DownloadRequirementsError):
+    """
+    Raised when there was an error that could not be fixed by
+    download-requirements script.
+    """
+
+
 class PackageNotfound(CriticalError):
     """
     Raised when there was no package found by the query tool.
     """
 
+
+class ChecksumMismatch(DownloadRequirementsError):
+    """
+    Raised when there was a file checksum mismatch.
+    """
     def __init__(self, msg: str):
-        super().__init__(msg)
+        super().__init__(f'{msg} - download failed due to checksum mismatch, '
+                         'WARNING someone might have replaced the file')

ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py

@@ -9,7 +9,7 @@
 from src.command.toolchain import Toolchain, TOOLCHAINS
 from src.config import Config
 from src.crypt import get_sha1, get_sha256
-from src.error import CriticalError
+from src.error import CriticalError, ChecksumMismatch
 
 
 def load_yaml_file(filename: Path) -> Any:
@@ -140,8 +140,12 @@ def __download_files(self, files: Dict[str, Dict], dest: Path):
 
                 logging.info(f'- {file}')
                 self._download_file(file, filepath)
+
+                if files[file]['sha256'] != get_sha256(filepath):
+                    raise ChecksumMismatch(f'- {file}')
+
             except CriticalError:
-                logging.warn(f'Could not download file: {file}')
+                logging.warning(f'Could not download file: {file}')
 
     def __download_grafana_dashboards(self):
         """
@@ -158,8 +162,12 @@ def __download_grafana_dashboards(self):
 
                 logging.info(f'- {dashboard}')
                 self._download_grafana_dashboard(dashboards[dashboard]['url'], output_file)
+
+                if dashboards[dashboard]['sha256'] != get_sha256(output_file):
+                    raise ChecksumMismatch(f'- {dashboard}')
+
             except CriticalError:
-                logging.warn(f'Could not download grafana dashboard: {dashboard}')
+                logging.warning(f'Could not download grafana dashboard: {dashboard}')
 
     def __download_crane(self):
         """
@@ -174,6 +182,10 @@ def __download_crane(self):
             logging.debug('crane - checksum ok, skipped')
         else:
             self._download_crane_binary(first_crane, crane_package_path)
+
+            if cranes[first_crane]['sha256'] != get_sha256(crane_package_path):
+                raise ChecksumMismatch('crane')
+
             self._tools.tar.unpack(crane_package_path, Path('crane'), directory=self._cfg.dest_dir)
             chmod(crane_path, 0o0755)
 
@@ -194,14 +206,24 @@ def _download_images(self):
                 url, version = image.split(':')
                 filename = Path(f'{url.split("/")[-1]}-{version}.tar')  # format: image_version.tar
 
-                if images[image]['sha1'] == get_sha1(self._cfg.dest_images / filename):
+                image_file = self._cfg.dest_images / filename
+                if images[image]['sha1'] == get_sha1(image_file):
                     logging.debug(f'- {image} - checksum ok, skipped')
                     continue
 
                 logging.info(f'- {image}')
-                self._tools.crane.pull(image, self._cfg.dest_images / filename, platform)
+                self._tools.crane.pull(image, image_file, platform)
+
+                if images[image]['sha1'] != get_sha1(image_file):
+                    try:
+                        if images[image]['allow_mismatch']:
+                            logging.warning(f'- {image} - allow_mismatch flag used, continue downloading')
+                            continue
+                    except KeyError:
+                        raise ChecksumMismatch(f'- {image}')
+
             except CriticalError:
-                logging.warn(f'Could not download image: `{image}`')
+                logging.warning(f'Could not download image: `{image}`')
 
     def _cleanup(self):
         """

ansible/playbooks/roles/repository/files/download-requirements/src/run.py

@@ -1,7 +1,6 @@
 from typing import Dict
 
 from src.config import Config, OSType
-from src.error import CriticalError
 from src.mode.base_mode import BaseMode
 from src.mode.debian_family_mode import DebianFamilyMode
 from src.mode.red_hat_family_mode import RedHatFamilyMode

schema/common/defaults/configuration/image-registry.yml

@@ -4,8 +4,8 @@ name: default
 specification:
   description: "Local registry with Docker images"
   registry_image:
-    name: "registry:2"
-    file_name: registry-2.tar
+    name: "registry:2.8.0"
+    file_name: registry-2.8.0.tar
   images_to_load:
     x86_64:
       generic: