diff --git a/core/src/epicli/data/common/ansible/playbooks/helm.yml b/core/src/epicli/data/common/ansible/playbooks/helm.yml index 92f3a8be05..b2d0693d68 100644 --- a/core/src/epicli/data/common/ansible/playbooks/helm.yml +++ b/core/src/epicli/data/common/ansible/playbooks/helm.yml @@ -1,8 +1,7 @@ --- - -- hosts: kubernetes_master[0] +- hosts: helm become: true become_method: sudo become_flags: --login roles: - - helm \ No newline at end of file + - helm diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml index f943e4753c..dc92616c23 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml @@ -1,5 +1,4 @@ --- - - name: "Download file {{ file_name }}" get_url: url: "{{ repository_url }}/files/{{ file_name }}" @@ -7,6 +6,6 @@ validate_certs: "{{ validate_certs }}" register: get_url_result until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" - retries: "3" - delay: "2" + retries: 3 + delay: 2 become: false diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml index d8e12f15ee..cf41411d62 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml @@ -1,12 +1,11 @@ --- - -- name: Download image {{ file_name }} +- name: "Download image {{ file_name }}" get_url: url: "{{ repository_url }}/images/{{ file_name }}" dest: "{{ download_directory }}" validate_certs: "{{ validate_certs }}" register: get_url_result until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" - retries: "3" - delay: "2" - become: false \ No newline at end of file + retries: 3 + delay: 2 + become: false diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_newest_file.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_newest_file.yml new file mode 100644 index 0000000000..c5266650d9 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_newest_file.yml @@ -0,0 +1,18 @@ +--- +# Invoke with (example): +#- set_fact: +# archive_name_regexp: '^helm-v([0-9.]+)-linux-amd64.tar.gz$' +# archive_version_format: '{:0>4s}.{:0>4s}.{:0>4s}' # OPTIONAL + +- name: Get "files" list from the repository + import_tasks: list_files.yml + +- name: Pick the newest archive file + import_tasks: pick_newest.yml + vars: + archive_names: "{{ list_files_result }}" + +- name: Download the newest archive file + import_tasks: download_file.yml + vars: + file_name: "{{ newest_archive_name }}" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/list_files.yml b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/list_files.yml new file mode 100644 index 0000000000..270fe93070 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/list_files.yml @@ -0,0 +1,25 @@ +--- +- name: Get file listing + uri: + method: GET + url: "{{ repository_url }}/files/?F=0" # F=0 formats the listing as a simple list (not FancyIndexed) + body_format: raw + return_content: true + validate_certs: "{{ validate_certs | default(false, true) | bool }}" # handling "undefined", "null", "empty" and "boolean" values all at once + register: uri_list_files + until: uri_list_files is success + retries: 3 + delay: 2 + become: false + +# TODO: make it work with yaml or json (instead of html, sic!). +- name: Parse html response and return file listing + set_fact: + list_files_result: >- + {{ lines | select('match', regexp) + | reject('match', '.*Parent Directory.*') + | map('regex_replace', regexp, '\1') + | list }} + vars: + lines: "{{ uri_list_files.content.splitlines() }}" + regexp: '.*
  • 4s}.{:0>4s}.{:0>4s}' # OPTIONAL + +- name: "Find and return the newest archive's name and version (using pattern: {{ archive_name_regexp }})" + + set_fact: + newest_archive_name: >- + {{ _newest_archive_name }} + + newest_archive_version: >- + {{ _newest_archive_name | regex_replace(archive_name_regexp, '\1') }} + + vars: + # Provide a default value for the version format (assuming 3 segments, left-padded up to 4 zeros). + _archive_version_format: >- + {{ archive_version_format | default( '{:0>4s}.{:0>4s}.{:0>4s}' ) }} + + # Drop all archive names not matching the pattern. + _archive_names_filtered: >- + {{ archive_names | select('match', archive_name_regexp) | list }} + + # Extract all archive version strings. + _archive_versions: >- + {{ _archive_names_filtered | map('regex_replace', archive_name_regexp, '\1') | list }} + + # Zero-pad each segment of the version string (returns a list of strings). + _archive_versions_padded: >- + {%- set output = [] -%} + {%- for version in _archive_versions -%} + {{- output.append( _archive_version_format.format(*version.split('.')) ) -}} + {%- endfor -%} + {{- output -}} + + # Create a version (padded) to archive name mapping. + _version_to_name_map: >- + {{ dict(_archive_versions_padded | zip(_archive_names_filtered)) }} + + # Use map to get the newest archive's name. + _newest_archive_name: >- + {{ _version_to_name_map[_version_to_name_map.keys() | max] }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/gather-facts.yml b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/gather-facts.yml new file mode 100644 index 0000000000..d79d805c3c --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/gather-facts.yml @@ -0,0 +1,8 @@ +--- +- name: Produce final list of hosts to deploy Helm chart sources on + set_fact: + target_repository_hostnames: >- + {{ groups.helm | difference(available_masters) | union([resolved_repository_hostname]) | unique }} + vars: + available_masters: >- + {{ groups.kubernetes_master | default([]) }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-bin.yml b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-bin.yml index 75ce5b6799..77002b1051 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-bin.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-bin.yml @@ -1,17 +1,40 @@ --- +- name: Download the newest Helm binary package from the epirepo + import_role: + name: download + tasks_from: download_newest_file + vars: + archive_name_regexp: '^helm-v([0-9.]+)-linux-amd64.tar.gz$' + validate_certs: false + +- name: Create versioned destination directory for Helm binary + file: + dest: "/usr/local/bin/helm-{{ newest_archive_version }}/" + state: directory + - name: Unpack Helm binary - unarchive: - src: "{{ specification.apache_epirepo_path }}/files/helm-{{ specification.version }}-linux-amd64.tar.gz" - dest: "/usr/local/bin" + unarchive: + src: "{{ download_directory }}/{{ newest_archive_name }}" + dest: "/usr/local/bin/helm-{{ newest_archive_version }}/" + creates: "/usr/local/bin/helm-{{ newest_archive_version }}/helm" extra_opts: - --strip=1 - --wildcards - '*/helm' - remote_src: yes + remote_src: true + register: unarchive_result -- name: Create symlink for Helm binary - file: - src: "/usr/local/bin/helm" - dest: /usr/local/sbin/helm - state: link - when: ansible_os_family == "RedHat" \ No newline at end of file +- when: unarchive_result is changed + block: + - name: Create symlink to enable the newest Helm binary + file: + src: "/usr/local/bin/helm-{{ newest_archive_version }}/helm" + dest: "/usr/local/bin/helm" + state: link + + - name: Create "sbin" symlink for Helm binary (RedHat fix) + file: + src: "/usr/local/bin/helm" + dest: "/usr/local/sbin/helm" + state: link + when: ansible_os_family == "RedHat" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-chart.yml b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-chart.yml index 9bcfe22edd..204febd625 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-chart.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-chart.yml @@ -1,24 +1,49 @@ --- -- name: Copy Helm charts custom configuration to file - copy: - content: "{{ helm_chart_values }}" - dest: "/tmp/{{ helm_chart_name }}_values.yaml" - when: helm_chart_values != "" +- name: Prepare configuration and upgrade/install Helm chart + vars: + # Handling "undefined", "null", "empty" and "boolean" values all at once. + disable_helm_chart_bool: "{{ disable_helm_chart | default(false, true) | bool }}" -- name: Install Helm chart - shell: helm upgrade --install -f /tmp/{{ helm_chart_name }}_values.yaml {{ helm_chart_name }} {{ helm_chart_repo_name }}/{{ helm_chart_name }} - when: - - disable_helm_chart != "" - - not disable_helm_chart - - helm_chart_values != "" + # Handling "undefined", "null" and "empty" values all at once. + helm_chart_values_bool: "{{ helm_chart_values | default(false) | ternary(true, false) }}" -- name: Delete Helm chart - shell: helm delete {{ helm_chart_name }} --purge - when: - - disable_helm_chart != "" - - disable_helm_chart + always: + - name: Clean up temporary Helm chart values file + file: + state: absent + path: /tmp/{{ helm_chart_name }}_values.yaml -- name: Clean up temporary Helm chart values file - file: - state: absent - path: /tmp/{{ helm_chart_name }}_values.yaml" \ No newline at end of file + block: + # IF `disable_helm_chart` + - when: disable_helm_chart_bool + block: + - name: Delete Helm chart + shell: | + helm delete --purge {{ helm_chart_name }} + + # ELSE + - when: not disable_helm_chart_bool + block: + # IF `helm_chart_values` + - when: helm_chart_values_bool + block: + - name: Copy Helm chart's custom configuration to file + copy: + content: "{{ helm_chart_values }}" + dest: /tmp/{{ helm_chart_name }}_values.yaml + + - name: Install Helm chart (with custom values.yaml) + shell: | + helm upgrade --install \ + -f /tmp/{{ helm_chart_name }}_values.yaml \ + {{ helm_chart_name }} \ + {{ helm_chart_repo_name }}/{{ helm_chart_name }} + + # ELSE + - when: not helm_chart_values_bool + block: + - name: Install Helm chart (with default values.yaml) + shell: | + helm upgrade --install \ + {{ helm_chart_name }} \ + {{ helm_chart_repo_name }}/{{ helm_chart_name }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/main.yml index 2bcbf6fe0c..27c626d0c0 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/main.yml @@ -1,41 +1,48 @@ --- -- name: Check if Helm chart repo already exists - shell: helm repo list | grep {{ helm_chart_repo_name }} - args: - executable: /bin/bash - register: helm_repo_list - failed_when: (helm_repo_list.rc != 0 and not 'no repositories' in helm_repo_list.stderr) - or helm_repo_list.rc > 1 +- import_tasks: gather-facts.yml -- name: Check if files exist in {{ specification.apache_epirepo_path }}/helm-charts/system - find: - paths: "{{ specification.apache_epirepo_path }}/helm-charts/system" - register: helm_charts_files_number - -- name: "Add Helm chart repo and install charts" +- name: Deploy the "system" Helm charts + when: inventory_hostname == target_repository_hostnames[0] # execute on the main repository host only block: - - name: Get Helm charts list from repo before update - shell: helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name' - register: helm_charts_list_before_update + - name: Check if Helm chart repo already exists + shell: | + helm repo list | grep {{ helm_chart_repo_name }} + register: helm_repo_list + failed_when: (helm_repo_list.rc != 0 and not 'no repositories' in helm_repo_list.stderr) + or helm_repo_list.rc > 1 + + - name: "Check if files exist in {{ specification.apache_epirepo_path }}/helm-charts/system" + find: + paths: "{{ specification.apache_epirepo_path }}/helm-charts/system" + register: helm_charts_files_number - - name: "Add {{ helm_chart_repo_name }} Helm repository from url {{ repository_url }}" - shell: helm repo add {{ helm_chart_repo_name }} {{ repository_url }}/helm-charts/system - when: - - helm_chart_repo_name not in helm_repo_list.stdout + - name: Add Helm chart repo and install charts + when: helm_charts_files_number.matched > 1 + block: + - name: Get Helm charts list from repo before update + shell: | + helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name' + register: helm_charts_list_before_update - - name: Update Helm repo - shell: helm repo update + - name: "Add {{ helm_chart_repo_name }} Helm repository from url {{ repository_url }}" + shell: | + helm repo add {{ helm_chart_repo_name }} {{ repository_url }}/helm-charts/system + when: + - helm_chart_repo_name not in helm_repo_list.stdout - - name: Get Helm charts list from repo after update - shell: helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name' - register: helm_charts_list_after_update + - name: Update Helm repo + shell: | + helm repo update - - name: Delete Helm charts not present in repository - shell: helm delete {{ chart_subname }} --purge - vars: - chart_subname: "{{ item.split('/')[1] }}" - loop: "{{ helm_charts_list_before_update.stdout_lines }}" - when: item not in helm_charts_list_after_update.stdout_lines - when: - - helm_charts_files_number.matched > 1 + - name: Get Helm charts list from repo after update + shell: | + helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name' + register: helm_charts_list_after_update + - name: Delete Helm charts not present in repository + shell: | + helm delete {{ chart_subname }} --purge + vars: + chart_subname: "{{ item.split('/')[1] }}" + loop: "{{ helm_charts_list_before_update.stdout_lines }}" + when: item not in helm_charts_list_after_update.stdout_lines diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/helm_charts/tasks/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/helm_charts/tasks/main.yml index 4ed67db71d..9a04335d8e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/helm_charts/tasks/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/helm_charts/tasks/main.yml @@ -1,12 +1,20 @@ --- -- name: Create Helm charts directory - file: - path: "{{ specification.apache_epirepo_path }}/helm-charts" - state: directory - mode: u=rwx,go=rx +- name: Borrow the fact initializer from the "repository" role + import_role: + name: repository + tasks_from: gather-facts -- name: Copy Helm charts to repository server - copy: - src: "./" - dest: "{{ specification.apache_epirepo_path }}/helm-charts" - mode: u=rwx,go=rx \ No newline at end of file +- name: Copy Helm chart sources (to repository hosts only) + when: inventory_hostname == target_repository_hostnames[0] # execute on the main repository host only + block: + - name: Create Helm charts directory + file: + path: "{{ specification.apache_epirepo_path }}/helm-charts" + state: directory + mode: u=rwx,go=rx + + - name: Copy Helm charts to repository server + copy: + src: "./" + dest: "{{ specification.apache_epirepo_path }}/helm-charts" + mode: u=rwx,go=rx diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/create-helm-repo.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/create-helm-repo.yml index 984c0e9628..99133ba9f7 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/create-helm-repo.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/create-helm-repo.yml @@ -1,10 +1,10 @@ --- -- name: Include Helm binary installation +- name: Include Helm binary installation include_role: name: helm tasks_from: install-bin -- name: Get list of directories for system Helm charts +- name: Get list of directories for system Helm charts find: paths: "{{ specification.apache_epirepo_path }}/helm-charts/system" file_type: directory @@ -32,4 +32,4 @@ loop: "{{ system_charts_dir_list.files }}" when: - not custom_repository_url - - system_charts_dir_list.matched > 0 + - system_charts_dir_list.matched > 0 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml index 696ad55133..019025f273 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/teardown.yml @@ -9,7 +9,7 @@ shell: /tmp/epi-repository-setup-scripts/disable-epirepo-client.sh - name: Remove epirepo packages directory - file: + file: state: absent path: "{{ apache_epirepo_path }}/packages" when: @@ -20,12 +20,12 @@ file: state: absent path: "{{ apache_epirepo_path }}/helm-charts" - when: + when: - specification.teardown.remove.helm_charts is defined - specification.teardown.remove.helm_charts - name: Remove Docker images from epirepo - file: + file: state: absent path: "{{ apache_epirepo_path }}/images" when: @@ -33,7 +33,7 @@ - specification.teardown.remove.images - name: Remove files directory from epirepo - file: + file: state: absent path: "{{ apache_epirepo_path }}/files" when: diff --git a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml index 71fe44e0d8..4eab8632c9 100644 --- a/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml +++ b/core/src/epicli/data/common/defaults/configuration/feature-mapping.yml @@ -13,6 +13,8 @@ specification: enabled: yes - name: kubernetes-node enabled: yes + - name: helm + enabled: yes - name: logging enabled: yes - name: opendistro-for-elasticsearch @@ -47,8 +49,6 @@ specification: enabled: yes - name: haproxy-exporter enabled: yes - - name: helm - enabled: yes - name: vault enabled: yes - name: applications diff --git a/core/src/epicli/data/common/defaults/configuration/helm-charts.yml b/core/src/epicli/data/common/defaults/configuration/helm-charts.yml index bdf1afa346..2be4d5c997 100644 --- a/core/src/epicli/data/common/defaults/configuration/helm-charts.yml +++ b/core/src/epicli/data/common/defaults/configuration/helm-charts.yml @@ -2,4 +2,4 @@ kind: configuration/helm-charts title: "Helm charts" name: default specification: - apache_epirepo_path: "/var/www/html/epirepo" \ No newline at end of file + apache_epirepo_path: "/var/www/html/epirepo" diff --git a/core/src/epicli/data/common/defaults/configuration/helm.yml b/core/src/epicli/data/common/defaults/configuration/helm.yml index ec37f564b4..54b1c5f010 100644 --- a/core/src/epicli/data/common/defaults/configuration/helm.yml +++ b/core/src/epicli/data/common/defaults/configuration/helm.yml @@ -2,5 +2,4 @@ kind: configuration/helm title: "Helm" name: default specification: - version: v3.2.0 - apache_epirepo_path: "/var/www/html/epirepo" \ No newline at end of file + apache_epirepo_path: "/var/www/html/epirepo" diff --git a/core/src/epicli/data/common/defaults/configuration/repository.yml b/core/src/epicli/data/common/defaults/configuration/repository.yml index edcb92221c..fdac785a3d 100644 --- a/core/src/epicli/data/common/defaults/configuration/repository.yml +++ b/core/src/epicli/data/common/defaults/configuration/repository.yml @@ -11,4 +11,4 @@ specification: files: false helm_charts: false images: false - packages: false + packages: false