Add image-registry configuration reading (#3106) (#3159)

* Add image-registry configuration reading (#3106)

ansible/playbooks/filter_plugins/container.py

@@ -0,0 +1,27 @@
+from typing import Any, Dict, List
+
+
+class FilterModule:
+    """ Filters for Python's container types """
+
+    def filters(self):
+        return {
+            'dict_to_list': self.dict_to_list
+        }
+
+    def dict_to_list(self, data: Dict, only_values: bool = False, only_keys: bool = False) -> List:
+        """
+        Convert dict to list without using Ansible's loop mechanism with dict2items filter.
+
+        :param data: to be converted into a list
+        :param only_values: construct list with only dict's values
+        :param only_keys: construct list with only dict's keys
+        :return: data transformed into a list
+        """
+        if only_values:
+            return list(data.values())
+
+        if only_keys:
+            return list(data.keys())
+
+        return list(data.items())

ansible/playbooks/roles/download/tasks/list_files.yml

@@ -1,25 +1,9 @@
 ---
-- name: Get file listing
-  uri:
-    method: GET
-    url: "{{ repository_url }}/files/?F=0"  # F=0 formats the listing as a simple list (not FancyIndexed)
-    body_format: raw
-    return_content: true
-    validate_certs: "{{ validate_certs | default(false, true) | bool }}"  # handling "undefined", "null", "empty" and "boolean" values all at once
-  register: uri_list_files
-  until: uri_list_files is success
-  retries: 3
-  delay: 2
-  become: false
+- name: Get files list from the repository
+  include_tasks: list_requirements.yml
+  vars:
+    _requirements: files
 
-# TODO: make it work with yaml or json (instead of html, sic!).
-- name: Parse html response and return file listing
+- name: Set files in repository as fact
   set_fact:
-    list_files_result: >-
-      {{ lines | select('match', regexp)
-               | reject('match', '.*Parent Directory.*')
-               | map('regex_replace', regexp, '\1')
-               | list }}
-  vars:
-    lines: "{{ uri_list_files.content.splitlines() }}"
-    regexp: '.*<li><a href="([^"]+)".*'
+    list_files_result: "{{ list_requirements_result }}"

ansible/playbooks/roles/download/tasks/list_images.yml

@@ -0,0 +1,9 @@
+---
+- name: Get images list from the repository
+  include_tasks: list_requirements.yml
+  vars:
+    _requirements: images
+
+- name: Set images in repository as fact
+  set_fact:
+    list_images_result: "{{ list_requirements_result }}"

ansible/playbooks/roles/download/tasks/list_requirements.yml

@@ -0,0 +1,25 @@
+---
+- name: Get requirements listing
+  uri:
+    method: GET
+    url: "{{ repository_url }}/{{ _requirements }}/?F=0"  # F=0 formats the listing as a simple list (not FancyIndexed)
+    body_format: raw
+    return_content: true
+    validate_certs: "{{ validate_certs | default(false, true) | bool }}"  # handling "undefined", "null", "empty" and "boolean" values all at once
+  register: uri_list_files
+  until: uri_list_files is success
+  retries: 3
+  delay: 2
+  become: false
+
+# TODO: make it work with yaml or json (instead of html, sic!).
+- name: Parse html response and return requirements listing
+  set_fact:
+    list_requirements_result: >-
+      {{ lines | select('match', regexp)
+               | reject('match', '.*Parent Directory.*')
+               | map('regex_replace', regexp, '\1')
+               | list }}
+  vars:
+    lines: "{{ uri_list_files.content.splitlines() }}"
+    regexp: '.*<li><a href="([^"]+)".*'

ansible/playbooks/roles/image_registry/tasks/main.yml

@@ -122,24 +122,59 @@
             --name {{ epiphany_registry.container_name }} -v {{ epiphany_registry.volume_name }}:/var/lib/registry
             {{ specification.registry_image.name }}
 
-    - name: Set images to load
+    - name: Define images to unpack
       set_fact:
-        generic_and_current_images: >-
-          {{ specification.images_to_load[ansible_architecture].generic + specification.images_to_load[ansible_architecture].current }}
-        legacy_images: "{{ specification.images_to_load[ansible_architecture].legacy }}"
+        current_schema_images: "{{ specification.images_to_load[ansible_architecture].current }}"
+        generic_schema_images: "{{ specification.images_to_load[ansible_architecture].generic }}"
+        legacy_schema_images: "{{ specification.images_to_load[ansible_architecture].legacy }}"
+
+    - name: Initialize image facts
+      set_fact:
+        requested_images: []
+        current_images: []
+        generic_images: []
+        legacy_images: []
+
+    - name: Set list of current images to be loaded/pushed
+      set_fact:
+        current_images: "{{ current_schema_images | dict_to_list(only_values='True') | flatten }}"
+
+    - name: Set list of generic images to be loaded/pushed
+      set_fact:
+        generic_images: "{{ generic_schema_images | dict_to_list(only_values='True') | flatten }}"
+
+    - name: Set list of legacy images to be loaded/pushed
+      set_fact:
+        legacy_images: "{{ legacy_schema_images | dict_to_list(only_values='True') | flatten }}"
+
+    - name: Merge current and generic images
+      set_fact:
+        current_and_generic_images: >-
+          {{ current_images + generic_images }}
+
+    - name: Get list of available images
+      include_role:
+        name: download
+        tasks_from: list_images.yml
+
+    - name: Filter only requested images
+      set_fact:  # gather only images listed in schema to avoid downloading unknown files
+        requested_images: "{{ requested_images + [item] }}"
+      when: "{{ item.file_name in list_images_result }}"
+      loop: "{{ current_and_generic_images }}"
 
     - name: Load generic and current version images
       vars:
         docker_image: "{{ item }}"
       include_tasks: load-image.yml
-      loop: "{{ generic_and_current_images }}"
+      loop: "{{ requested_images }}"
 
     - name: Push generic and current version images to registry
       vars:
         docker_image: "{{ item }}"
         new_image_tag: "{{ image_registry_address }}/{{ item.name }}"
       include_tasks: push-image.yml
-      loop: "{{ generic_and_current_images }}"
+      loop: "{{ requested_images }}"
 
     - name: Load legacy version images to registry when upgrading
       when: is_upgrade_run