diff --git a/ansible/roles/sf-firehose/defaults/main.yml b/ansible/roles/sf-firehose/defaults/main.yml deleted file mode 100644 index b8382ecc6..000000000 --- a/ansible/roles/sf-firehose/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -role_actions: - - install - - setup - -role_package: firehose diff --git a/ansible/roles/sf-firehose/meta/main.yml b/ansible/roles/sf-firehose/meta/main.yml deleted file mode 100644 index 0775795cb..000000000 --- a/ansible/roles/sf-firehose/meta/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -galaxy_info: - description: Firehose service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory -dependencies: - - role: sf-mosquitto diff --git a/ansible/roles/sf-firehose/tasks/get_logs.yml b/ansible/roles/sf-firehose/tasks/get_logs.yml deleted file mode 100644 index ed97d539c..000000000 --- a/ansible/roles/sf-firehose/tasks/get_logs.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/ansible/roles/sf-firehose/tasks/install.yml b/ansible/roles/sf-firehose/tasks/install.yml deleted file mode 100644 index ed97d539c..000000000 --- a/ansible/roles/sf-firehose/tasks/install.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/ansible/roles/sf-firehose/tasks/main.yml b/ansible/roles/sf-firehose/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-firehose/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-firehose/tasks/setup.yml b/ansible/roles/sf-firehose/tasks/setup.yml deleted file mode 100644 index ed97d539c..000000000 --- a/ansible/roles/sf-firehose/tasks/setup.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/ansible/roles/sf-gerritbot/tasks/setup.yml b/ansible/roles/sf-gerritbot/tasks/setup.yml index 6c46e5998..bb6d0249b 100644 --- a/ansible/roles/sf-gerritbot/tasks/setup.yml +++ b/ansible/roles/sf-gerritbot/tasks/setup.yml @@ -74,8 +74,6 @@ register: _gerritbot_channel changed_when: false failed_when: false - args: - warn: false - name: Install a dummy channels.yaml file copy: diff --git a/ansible/roles/sf-grafana/defaults/main.yml b/ansible/roles/sf-grafana/defaults/main.yml deleted file mode 100644 index b0924d448..000000000 --- a/ansible/roles/sf-grafana/defaults/main.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -role_actions: - - disable - - install - - setup - - postconf - - config_update - -role_package: grafana -container_name: grafana-oss - -owner: grafana -group: grafana -container_uid: 472 -container_gid: 0 - -grafana_config_dir: /etc/grafana -grafana_lib_dir: /var/lib/grafana -grafana_log_dir: /var/log/grafana - -grafana_container_tag: "9.2.6" -grafana_container_version: "{{ grafana_container_tag }}-1" -grafyaml_container: "quay.io/software-factory/grafyaml:0.0.9-1" - -# Configurable directories https://grafana.com/docs/grafana/latest/administration/configure-docker/#default-paths -grafana_components: - - service: "grafana" - image: "quay.io/software-factory/{{ container_name }}:{{ grafana_container_version }}" - params: >- - --network host - --user {{ container_uid }}:{{ container_gid }} - --uidmap {{ container_uid }}:{{ grafana_uid | default(1000) }}:1 - --gidmap 0:{{ grafana_gid | default(1000) }}:1 - --uidmap 0:4000:471 - --gidmap 1:4000:999 - --env GF_INSTALL_PLUGINS={{ grafana_plugins }} - --volume {{ grafana_config_dir }}:{{ grafana_config_dir }}:z - --volume {{ grafana_lib_dir }}:{{ grafana_lib_dir }}:z - --volume {{ grafana_log_dir }}:{{ grafana_log_dir }}:z - --volume /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt - {% if sf_grafana_additional_params is defined %} - {{ sf_grafana_additional_params }} - {% endif %} - -# Grafana Container Plugins Docs: https://grafana.com/docs/grafana/latest/installation/docker/#build-and-run-a-docker-image-with-pre-installed-plugins -# Grafan Plugins page: https://grafana.com/grafana/plugins/ -grafana_plugins: "grafana-clock-panel" - -grafana_repo_name: grafana -grafana_repo_desc: Grafana repository -grafana_repo_url: https://packages.grafana.com/oss/rpm -grafana_repo_gpgkey: https://packages.grafana.com/gpg.key -grafana_repo_gpgcheck: 1 - -grafana_database_type: mysql -grafana_mysql_host: mysql.example.com -grafana_mysql_port: 3306 -grafana_mysql_db: grafana -grafana_mysql_user: grafana -grafana_mysql_password: CHANGE_ME - -grafana_http_port: 3000 -grafana_host: grafana.example.com -grafana_internal_url: "http://{{ grafana_host }}:{{ grafana_http_port }}" -grafana_internal_datasource_url: '{{ grafana_internal_url }}/api/datasources' -grafana_internal_apikey_url: '{{ grafana_internal_url }}/api/auth/keys' -grafana_server_root_url: "{{ gateway_url }}/grafana/" -# When using bool with ini_file, True or False should be between quotes for it's not an ansible bool but a string -# The result on the ini file is a bool -# https://docs.ansible.com/ansible/latest/modules/ini_file_module.html#parameter-value -grafana_users_allow_sign_up: 'False' -grafana_auth_anonymous_enabled: 'True' -grafana_auth_anonymous_org_role: Viewer -grafana_auth_basic_enabled: 'True' -grafana_auth_proxy_enabled: 'True' -grafana_auth_proxy_header_name: X-Forwarded-User -grafana_auth_proxy_header_property: username -grafana_auth_proxy_auto_sign_up: 'True' -grafana_admin_password: CHANGE_ME - -influxdb_host: influxdb.example.com -telegraf_influxdb_user: telegraf -telegraf_influxdb_password: CHANGE_ME - -service_name: grafana -service_real_name: grafana-server - -grafyaml_metrics: /var/lib/software-factory/metrics - -grafana_data_source: - isDefault: true - database: telegraf - name: influxdb - type: influxdb - user: '{{ telegraf_influxdb_user }}' - secureJsonData: - password: '{{ telegraf_influxdb_password }}' - url: 'https://{{ influxdb_host }}:8086' - access: proxy - basicAuth: false - -grafyaml_apikey_data: - name: 'grafyaml' - role: 'Admin' - -keycloak_default_realm: SF diff --git a/ansible/roles/sf-grafana/handlers/main.yml b/ansible/roles/sf-grafana/handlers/main.yml deleted file mode 100644 index 1bc93e74e..000000000 --- a/ansible/roles/sf-grafana/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: restart grafana - service: - name: grafana-server - state: restarted - failed_when: false diff --git a/ansible/roles/sf-grafana/meta/main.yml b/ansible/roles/sf-grafana/meta/main.yml deleted file mode 100644 index 14e21812d..000000000 --- a/ansible/roles/sf-grafana/meta/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -galaxy_info: - description: Grafana service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory diff --git a/ansible/roles/sf-grafana/meta/sfconfig.py b/ansible/roles/sf-grafana/meta/sfconfig.py deleted file mode 100644 index f6e985fb9..000000000 --- a/ansible/roles/sf-grafana/meta/sfconfig.py +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sfconfig.components import Component - - -class Grafana(Component): - def configure(self, args, host): - args.glue["grafana_internal_url"] = "http://%s:%s" % ( - args.glue["grafana_host"], args.defaults["grafana_http_port"]) - self.add_mysql_database(args, "grafana") diff --git a/ansible/roles/sf-grafana/tasks/config_update.yml b/ansible/roles/sf-grafana/tasks/config_update.yml deleted file mode 100644 index 38252030a..000000000 --- a/ansible/roles/sf-grafana/tasks/config_update.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Create metrics_config file - file: - path: /var/lib/software-factory/state/metrics_config - state: touch - changed_when: False - -- name: Check local metrics config - command: cat /var/lib/software-factory/state/metrics_config - changed_when: False - register: localconfig - -- name: Check upstream metrics config - command: git log -n 3 --oneline metrics/ - args: - chdir: /root/config - changed_when: False - register: upstreamconfig - -- name: Update metrics dashboards - shell: > - find . -regextype posix-egrep -regex '.*.(yaml|yml)$' | - xargs -I yaml grafana-dashboard update yaml - args: - chdir: "/root/config/metrics" - delegate_to: '{{ install_server_host }}' - when: localconfig.stdout != upstreamconfig.stdout - -- name: Write config repo sha1 matching current metrics configuration - copy: - content: "{{ upstreamconfig.stdout }}" - dest: /var/lib/software-factory/state/metrics_config - when: localconfig.stdout != upstreamconfig.stdout diff --git a/ansible/roles/sf-grafana/tasks/disable.yml b/ansible/roles/sf-grafana/tasks/disable.yml deleted file mode 100644 index a25f6a83d..000000000 --- a/ansible/roles/sf-grafana/tasks/disable.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- name: Disable services - service: - name: 'grafana-server' - state: stopped - enabled: "no" - ignore_errors: "yes" - -- block: - - name: Delete container - grafana - include_role: - name: sf-container - tasks_from: delete.yaml - loop: "{{ grafana_components }}" - - - name: Remove files - file: - path: "{{ item }}" - state: absent - loop: - - '/etc/{{ service_name }}' - - '/var/lib/{{ service_name }}' - - '/var/log/{{ service_name }}' - - "/etc/yum.repos.d/{{ grafana_repo_name }}.repo" - - '/etc/grafyaml' - - '/var/run/grafana' - - '{{ grafyaml_metrics }}' - - '/usr/bin/update-grafana-graphs.sh' - ignore_errors: "yes" - - - name: Remove packages - yum: - name: '{{ item }}' - state: absent - loop: - - '{{ service_name }}' - - 'grafyaml' - when: erase == True diff --git a/ansible/roles/sf-grafana/tasks/grafana_configuration.yml b/ansible/roles/sf-grafana/tasks/grafana_configuration.yml deleted file mode 100644 index 716eff675..000000000 --- a/ansible/roles/sf-grafana/tasks/grafana_configuration.yml +++ /dev/null @@ -1,154 +0,0 @@ ---- -- name: Wait for db initialization - wait_for: - port: "{{ grafana_http_port }}" - host: '{{ grafana_host }}' - -- name: Delete old datasource when fqdn is updated - uri: - url: '{{ grafana_internal_datasource_url }}/name/{{ grafana_data_source.name }}' - method: DELETE - user: admin - password: '{{ grafana_admin_password }}' - force_basic_auth: "yes" - ignore_errors: true - -- name: Get datasources - uri: - url: '{{ grafana_internal_datasource_url }}' - method: GET - user: admin - password: '{{ grafana_admin_password }}' - force_basic_auth: "yes" - register: datasources - -- name: set datasource fact - set_fact: - grafana_datasources: "{{ datasources.json | default([]) | map(attribute='name') | list }}" - -- name: Configure datasource - uri: - url: '{{ grafana_internal_datasource_url }}' - method: POST - user: admin - password: '{{ grafana_admin_password }}' - body: '{{ grafana_data_source | to_json }}' - body_format: json - force_basic_auth: "yes" - when: grafana_data_source.name not in grafana_datasources - -- name: Create grafyaml config directory - file: - path: /etc/grafyaml - mode: '0700' - state: directory - delegate_to: '{{ install_server_host }}' - -- name: Get all api keys - uri: - url: '{{ grafana_internal_apikey_url }}' - method: GET - user: admin - password: '{{ grafana_admin_password }}' - force_basic_auth: "yes" - register: grafana_apikeys - -- name: Create a list with all api key names - set_fact: - grafana_apikeys_names: "{{ grafana_apikeys.json | map(attribute='name') | list }}" - when: grafana_apikeys.json is defined - -- include_vars: - file: /var/lib/software-factory/bootstrap-data/secrets.yaml - name: grafana_secrets - when: grafyaml_apikey_data.name in grafana_apikeys_names - -- name: Validate existing grafyaml key - uri: - url: '{{ grafana_internal_datasource_url }}' - method: GET - force_basic_auth: "yes" - status_code: 200, 401 - headers: - Authorization: "Bearer {{ grafana_secrets.grafyaml_apikey }}" - register: grafyaml_check_apikeys - when: grafyaml_apikey_data.name in grafana_apikeys_names - -- name: Delete outdated grafyaml key if needed - block: - - block: - - name: Get all api keys ids - uri: - url: '{{ grafana_internal_apikey_url }}' - method: GET - user: admin - password: '{{ grafana_admin_password }}' - body_format: json - force_basic_auth: "yes" - register: grafana_get_apikeys - - - name: Delete outdated grafyaml apikey - uri: - url: '{{ grafana_internal_apikey_url }}/{{ item.id }}' - method: DELETE - user: admin - password: '{{ grafana_admin_password }}' - body_format: json - force_basic_auth: "yes" - loop: '{{ grafana_get_apikeys.json }}' - no_log: "{{ show_hidden_logs }}" - when: item.name == grafyaml_apikey_data.name - - - set_fact: - grafyaml_new_api_key: True - when: grafyaml_check_apikeys.status == 401 - when: grafyaml_check_apikeys is defined and - grafyaml_apikey_data.name in grafana_apikeys_names - -- name: Create api key and configuration file for grafyaml - block: - - name: Create api for grafyaml service - uri: - url: '{{ grafana_internal_apikey_url }}' - method: POST - user: admin - password: '{{ grafana_admin_password }}' - force_basic_auth: "yes" - body: '{{ grafyaml_apikey_data | to_json }}' - body_format: json - register: grafana_apikey - - - set_fact: - grafyaml_apikey: '{{ grafana_apikey.json.key }}' - - - name: Copy grafyaml_apikey in secrets.yaml - lineinfile: - path: /var/lib/software-factory/bootstrap-data/secrets.yaml - regexp: '^grafyaml_apikey' - line: 'grafyaml_apikey: {{ grafyaml_apikey }}' - when: grafana_apikeys_names is not defined or - grafyaml_apikey_data.name not in grafana_apikeys_names or - grafyaml_new_api_key is defined - delegate_to: '{{ install_server_host }}' - -- include_vars: - file: /var/lib/software-factory/bootstrap-data/secrets.yaml - when: grafyaml_apikey is not defined - -- name: Configure grafyaml - template: - src: grafyaml.conf.j2 - dest: /etc/grafyaml/grafyaml.conf - owner: root - group: root - mode: '0400' - delegate_to: '{{ install_server_host }}' - -- name: Setup cron for auto refresh of grafana zuul/nodepool dashboards - cron: - name: "Refresh grafana zuul/nodepool dashboards" - minute: '0' - hour: '0' - day: '*' - job: "/usr/bin/update-grafana-graphs.sh &> /dev/null" - delegate_to: '{{ install_server_host }}' diff --git a/ansible/roles/sf-grafana/tasks/install.yml b/ansible/roles/sf-grafana/tasks/install.yml deleted file mode 100644 index 880a82411..000000000 --- a/ansible/roles/sf-grafana/tasks/install.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- name: Check if CentOS system - set_fact: - extra_repos: "base,updates" - when: ansible_distribution == "CentOS" - -- name: Check if RedHat system - set_fact: - extra_repos: "rhel-7-server-rpms" - when: ansible_distribution == "RedHat" - -- name: check if grafana package is present - command: rpm -ql grafana - register: _grafana_package - changed_when: false - failed_when: false - args: - warn: false - -- name: Stop grafana-server service - systemd: - name: grafana-server - state: stopped - when: _grafana_package.rc == 0 - -- name: Remove grafana package - yum: - name: - - grafana - - grafyaml - state: absent - -- name: Reload systemd - systemd: - daemon_reload: yes - when: _grafana_package.rc == 0 - -- include_tasks: "{{ sf_tasks_dir }}/create_user.yml" - vars: - user_name: "{{ owner }}" - -- name: Pull image - include_role: - name: sf-container - tasks_from: install.yaml - loop: "{{ grafana_components }}" diff --git a/ansible/roles/sf-grafana/tasks/main.yml b/ansible/roles/sf-grafana/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-grafana/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-grafana/tasks/postconf.yml b/ansible/roles/sf-grafana/tasks/postconf.yml deleted file mode 100644 index c22c1d9df..000000000 --- a/ansible/roles/sf-grafana/tasks/postconf.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Ensure zuul-web is available - uri: - url: "http://{{ zuul_web_host }}:9000" - method: GET - status_code: "200" - retries: 6 - delay: 10 - -- name: Ensure grafana is available - uri: - url: "http://{{ grafana_host }}:3000" - method: GET - status_code: "200" - retries: 6 - delay: 10 - -- name: Delete old dashboards - shell: > - podman exec -it mysql sh -c "mysql -u{{ grafana_mysql_user }} -p{{ grafana_mysql_password }} grafana -e 'delete from dashboard where not slug=\"general-alerting\";'" - delegate_to: '{{ mysql_host }}' - when: - - sf_previous_version < 3.8 - -- name: Render dynamic graph - command: update-grafana-graphs.sh - changed_when: false - delegate_to: localhost diff --git a/ansible/roles/sf-grafana/tasks/setup.yml b/ansible/roles/sf-grafana/tasks/setup.yml deleted file mode 100644 index 76b9bfe14..000000000 --- a/ansible/roles/sf-grafana/tasks/setup.yml +++ /dev/null @@ -1,179 +0,0 @@ ---- -- name: Check container image version - include_role: - name: sf-container - tasks_from: check_version.yaml - loop: "{{ grafana_components }}" - -- name: Create container directory - include_role: - name: sf-container - tasks_from: create_directory.yaml - loop: - - {path: "{{ grafana_config_dir }}", mode: "0755", recurse: "yes"} - - {path: "{{ grafana_lib_dir }}", mode: "0775", recurse: "yes"} - - {path: "{{ grafana_log_dir }}", mode: "0755", recurse: "yes"} - -- name: Apply selinux port labelling - seport: - ports: "{{ grafana_http_port }}" - proto: tcp - setype: http_port_t - state: present - when: - - ansible_virtualization_type not in ["podman", "systemd-nspawn"] - - ansible_selinux.status == "enabled" - -- name: Configure grafana.ini - ini_file: - dest: "{{ grafana_config_dir }}/grafana.ini" - section: '{{ item.section }}' - option: '{{ item.option }}' - value: '{{ item.value }}' - loop: - - {section: server, option: http_port, value: '{{ grafana_http_port }}'} - - {section: server, option: root_url, value: '{{ grafana_server_root_url }}'} - - {section: database, option: type, value: '{{ grafana_database_type }}'} - - {section: database, option: host, value: '{{ grafana_mysql_host }}:{{ grafana_mysql_port }}'} - - {section: database, option: name, value: '{{ grafana_mysql_db }}'} - - {section: database, option: user, value: '{{ grafana_mysql_user }}'} - - {section: database, option: password, value: '{{ grafana_mysql_password }}'} - - {section: users, option: allow_sign_up, value: '{{ grafana_users_allow_sign_up }}'} - - {section: auth.anonymous, option: enabled, value: '{{ grafana_auth_anonymous_enabled }}'} - - {section: auth.anonymous, option: org_role, value: '{{ grafana_auth_anonymous_org_role }}'} - - {section: security, option: admin_password, value: '{{ grafana_admin_password }}'} - register: _grafana_ini - no_log: "{{ show_hidden_logs }}" - -- name: Configure authentication - ini_file: - dest: "{{ grafana_config_dir }}/grafana.ini" - section: '{{ item.section }}' - option: '{{ item.option }}' - value: '{{ item.value }}' - loop: - - {section: auth.generic_oauth, option: enabled, value: 'true'} - - {section: auth.generic_oauth, option: name, value: 'Software Factory SSO'} - - {section: auth.generic_oauth, option: client_id, value: 'grafana'} - - {section: auth.generic_oauth, option: client_secret, value: '{{ keycloak_grafana_client_secret }}'} - - {section: auth.generic_oauth, option: scopes, value: 'openid;profile'} - - {section: auth.generic_oauth, option: auth_url, value: 'https://{{ fqdn }}/auth/realms/{{ keycloak_default_realm }}/protocol/openid-connect/auth'} - - {section: auth.generic_oauth, option: token_url, value: 'https://{{ fqdn }}/auth/realms/{{ keycloak_default_realm }}/protocol/openid-connect/token'} - - {section: auth.generic_oauth, option: api_url, value: 'https://{{ fqdn }}/auth/realms/{{ keycloak_default_realm }}/protocol/openid-connect/userinfo'} - - {section: auth.generic_oauth, option: tls_client_ca, value: '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'} - register: _grafana_ini_auth - no_log: "{{ show_hidden_logs }}" - -# See https://community.grafana.com/t/grafana-will-not-start-after-update-from-4-1-3-to-4-3-6/5212/2 -# and https://github.com/puppetlabs/puppetlabs-pe_metrics_dashboard/issues/12 -- name: Fix /var/run/grafana issues - file: - path: /var/run/grafana - owner: grafana - group: grafana - mode: '0755' - state: directory - -- name: Get {{ owner }} uid - command: id -u {{ owner }} - register: _grafana_id - -- name: Get {{ owner }} gid - command: id -g {{ owner }} - register: _grafana_gid - -- name: Setting {{ owner }} uid var - set_fact: - grafana_uid: "{{ _grafana_id.stdout }}" - grafana_gid: "{{ _grafana_gid.stdout }}" - -- name: Create container - include_role: - name: sf-container - tasks_from: setup.yaml - loop: "{{ grafana_components }}" - -- name: Start service - systemd: - name: "{{ role_package }}" - state: started - daemon_reload: "yes" - enabled: "yes" - -- name: Wait for grafana to be ready - uri: - url: "{{ grafana_internal_url }}/api/health" - method: GET - status_code: 200 - return_content: yes - register: _grafana_health - retries: 30 - delay: 5 - until: - - _grafana_health.status == 200 - - _grafana_health.json.database == 'ok' - -- name: Fix systemd file to wait for mysql - ini_file: - dest: /lib/systemd/system/grafana-server.service - section: Service - option: ExecStartPre - value: /usr/libexec/software-factory/wait4mariadb - -- name: Create scripts to manage dashboards creation and update - block: - - name: Create /bin/update-graphs to manage grafana dashboards - copy: - dest: /bin/update-graphs - content: | - #!/bin/bash - set -e - find /var/lib/software-factory/metrics -regextype posix-egrep -regex '.*.(yaml|yml)$' | xargs -I file grafana-dashboard update file - mode: "0550" - - - name: Create /bin/grafana-dashboard to interact with container - copy: - dest: /bin/grafana-dashboard - content: | - #!/bin/sh -e - exec podman run --rm -v {{ grafyaml_metrics }}:{{ grafyaml_metrics }}:z \ - -v /etc/grafyaml/grafyaml.conf:/etc/grafyaml/grafyaml.conf:z \ - -v /bin/update-graphs:/bin/update-graphs:z \ - -v /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem \ - --env REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-bundle.crt \ - {{ grafyaml_container }} /bin/update-graphs - mode: "0550" - - - name: Install update-grafana-graphs script - template: - src: update-grafana-graphs.sh.j2 - dest: /usr/bin/update-grafana-graphs.sh - owner: root - group: root - mode: '0550' - - - name: Create metrics directory on the install server - file: - path: '{{ grafyaml_metrics }}' - state: directory - mode: '0755' - owner: '{{ owner }}' - group: '{{ group }}' - delegate_to: '{{ install_server_host }}' - -- name: Ensure service {{ role_package }} is enabled - systemd: - name: "{{ role_package }}" - state: started - daemon_reload: "yes" - enabled: "yes" - -- name: Restart {{ role_package }} container if config files changed - systemd: - name: "{{ role_package }}" - state: restarted - when: - - _grafana_ini is changed - - _grafana_ini_auth is changed - -- include_tasks: grafana_configuration.yml diff --git a/ansible/roles/sf-grafana/tasks/upgrade.yml b/ansible/roles/sf-grafana/tasks/upgrade.yml deleted file mode 100644 index 85274a414..000000000 --- a/ansible/roles/sf-grafana/tasks/upgrade.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- service_facts: - run_once: true - -- name: Upgrade container - include_role: - name: sf-container - tasks_from: upgrade.yaml - loop: "{{ grafana_components }}" diff --git a/ansible/roles/sf-grafana/templates/grafyaml.conf.j2 b/ansible/roles/sf-grafana/templates/grafyaml.conf.j2 deleted file mode 100644 index 528e692b4..000000000 --- a/ansible/roles/sf-grafana/templates/grafyaml.conf.j2 +++ /dev/null @@ -1,14 +0,0 @@ -[cache] -# Directory used by grafyaml to store its cache files. (string value) -#cachedir = ~/.cache/grafyaml - -# Maintain a special cache that contains an MD5 of every generated -# dashboard. (boolean value) -#enabled = true - -[grafana] -# URL for grafana server. (string value) -url = {{ grafana_internal_url }} - -# API key for access grafana. (string value) -apikey = {{ grafyaml_apikey }} diff --git a/ansible/roles/sf-grafana/templates/update-grafana-graphs.sh.j2 b/ansible/roles/sf-grafana/templates/update-grafana-graphs.sh.j2 deleted file mode 100755 index 2971b878f..000000000 --- a/ansible/roles/sf-grafana/templates/update-grafana-graphs.sh.j2 +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -{% if zuul_web_url|default(False) %} -sf-graph-render --zuul-url {{ zuul_web_url }} \ - --config-dir "/root/config" \ - --output-dir "{{ grafyaml_metrics }}" \ - update-grafyaml - -RET="$?" -if [ "${RET}" -ne "4" ] && [ "${RET}" -ne "0" ]; then - echo "Graph render failed" - exit 1 -elif [ "${RET}" -eq "4" ]; then - set -e - set -o pipefail - grafana-dashboard -fi -{% endif %} diff --git a/ansible/roles/sf-influxdb/defaults/main.yml b/ansible/roles/sf-influxdb/defaults/main.yml deleted file mode 100644 index 3440e428d..000000000 --- a/ansible/roles/sf-influxdb/defaults/main.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -role_actions: - - disable - - install - - setup - - get_logs - -role_package: influxdb - -container_uid: 1000 -container_gid: 1000 -owner: "{{ role_package }}" -group: "{{ role_package }}" - -influxdb_host: influxdb.example.com -influxdb_port: 8086 -influxdb_admin_password: CHANGE_ME -telegraf_influxdb_password: CHANGE_ME - -influxdb_repo_name: influxdb -influxdb_repo_desc: InfluxDB Repository -influxdb_repo_url: https://repos.influxdata.com/rhel/\$releasever/\$basearch/stable -influxdb_repo_gpgkey: https://repos.influxdata.com/influxdb.key -influxdb_repo_gpgcheck: 1 - -# Temporary pin the version to 1.8.6 due multiple bugs -# related to new version, e.g.: https://github.com/influxdata/influxdb/issues/21960 -influxdb_version: 1.8.6 -influxdb_container_version: "{{ influxdb_version }}-2" - -influxdb_config_dir: /etc/influxdb -influxdb_log_dir: /var/log/influxdb -influxdb_data_dir: /var/lib/influxdb - -influxdb_components: - - service: "{{ role_package }}" - image: "quay.io/software-factory/{{ role_package }}:{{ influxdb_container_version }}" - params: >- - --network host - --uidmap 0:4000:1000 - --gidmap 0:4000:1000 - --uidmap {{ container_uid }}:{{ influxdb_uid | default(1000) }}:1 - --gidmap {{ container_gid }}:{{ influxdb_gid | default(1000) }}:1 - --volume {{ influxdb_config_dir }}:{{ influxdb_config_dir }}:Z - --volume {{ influxdb_log_dir }}:{{ influxdb_log_dir }}:Z - --volume {{ influxdb_data_dir }}:{{ influxdb_data_dir }}:Z - --volume /etc/ssl/certs/ca-bundle.crt:/etc/ssl/certs/ca-bundle.crt - {% if sf_influxdb_additional_params is defined %} - {{ sf_influxdb_additional_params }} - {% endif %} diff --git a/ansible/roles/sf-influxdb/handlers/main.yml b/ansible/roles/sf-influxdb/handlers/main.yml deleted file mode 100644 index 11bc22a82..000000000 --- a/ansible/roles/sf-influxdb/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: restart influxdb - service: - name: influxdb - state: restarted diff --git a/ansible/roles/sf-influxdb/meta/main.yml b/ansible/roles/sf-influxdb/meta/main.yml deleted file mode 100644 index 062d7b43f..000000000 --- a/ansible/roles/sf-influxdb/meta/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -galaxy_info: - description: Influxdb service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory diff --git a/ansible/roles/sf-influxdb/meta/sfconfig.py b/ansible/roles/sf-influxdb/meta/sfconfig.py deleted file mode 100644 index 9c3df3130..000000000 --- a/ansible/roles/sf-influxdb/meta/sfconfig.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sfconfig.components import Component - - -class Influxdb(Component): - def configure(self, args, host): - self.get_or_generate_cert(args, "influxdb", host["hostname"]) diff --git a/ansible/roles/sf-influxdb/tasks/disable.yml b/ansible/roles/sf-influxdb/tasks/disable.yml deleted file mode 100644 index ff76d2227..000000000 --- a/ansible/roles/sf-influxdb/tasks/disable.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- name: Disable services - service: - name: '{{ item }}' - state: stopped - enabled: "no" - loop: - - '{{ service_real_name }}' - - '{{ role_package }}' - ignore_errors: "yes" - -- block: - - name: "Check if {{ role_package }} container exists" - command: "podman container exists {{ role_package }}" - ignore_errors: yes - register: _container_exists - changed_when: no - failed_when: _container_exists.rc not in [0, 1] - become: true - - - name: Delete containers - include_role: - name: sf-container - tasks_from: delete.yaml - loop: "{{ influxdb_components }}" - when: _container_exists.rc == 0 - - - name: Remove files - file: - path: "{{ item }}" - state: absent - loop: - - '/etc/{{ role_package }}' - - '/var/lib/{{ role_package }}' - - '/var/log/{{ role_package }}' - - "/etc/yum.repos.d/{{ influxdb_repo_name }}.repo" - - - name: Remove packages - yum: - name: '{{ item }}' - state: absent - loop: - - '{{ role_package }}' - when: _container_exists.rc != 0 - - when: erase == True diff --git a/ansible/roles/sf-influxdb/tasks/get_logs.yml b/ansible/roles/sf-influxdb/tasks/get_logs.yml deleted file mode 100644 index f66b897fd..000000000 --- a/ansible/roles/sf-influxdb/tasks/get_logs.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Getting container logs - include_role: - name: sf-container - tasks_from: get_logs.yaml - loop: - - container: "{{ role_package }}" - options: "" - target: "{{ log_dest }}" diff --git a/ansible/roles/sf-influxdb/tasks/influxdb_configuration.yml b/ansible/roles/sf-influxdb/tasks/influxdb_configuration.yml deleted file mode 100644 index 4329a6a20..000000000 --- a/ansible/roles/sf-influxdb/tasks/influxdb_configuration.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -- name: Wait for db initialization - wait_for: - port: 8086 - host: '{{ influxdb_host }}' - -- name: Set influxdb cli fact - set_fact: - influxdb_cli: 'podman exec -it {{ role_package }} influx -ssl -host {{ influxdb_host }} -port 8086' - -- name: Check if admin password is set - shell: '{{ influxdb_cli }} -execute "SHOW USERS"' - register: admin_password - failed_when: false - changed_when: false - no_log: "{{ show_hidden_logs }}" - -- name: Create admin user - shell: "{{ influxdb_cli }} -execute \"CREATE USER admin WITH PASSWORD '{{ influxdb_admin_password }}' WITH ALL PRIVILEGES\"" - when: "'create admin user' in admin_password.stdout" - no_log: "{{ show_hidden_logs }}" - -- name: Set influxdb cli fact - set_fact: - influxdb_cli: '{{ influxdb_cli }} -username admin -password {{ influxdb_admin_password }}' - -- name: Check if telegraf db exists - shell: '{{ influxdb_cli }} -execute "SHOW DATABASES"' - register: databases - changed_when: false - no_log: "{{ show_hidden_logs }}" - -- name: Create telegraf db - shell: '{{ influxdb_cli }} -execute "CREATE DATABASE telegraf"' - when: "'telegraf' not in databases.stdout" - -- name: Check if telegraf user exists - shell: '{{ influxdb_cli }} -execute "SHOW USERS"' - register: users - changed_when: false - no_log: "{{ show_hidden_logs }}" - -- block: - - name: Create telegraf user - shell: "{{ influxdb_cli }} -execute \"CREATE USER telegraf WITH PASSWORD '{{ telegraf_influxdb_password }}'\"" - - - name: Grant access for telegraf user - shell: "{{ influxdb_cli }} -execute 'GRANT ALL ON \"telegraf\" TO \"telegraf\"'" - ignore_errors: true - no_log: "{{ show_hidden_logs }}" - when: "'telegraf' not in users.stdout" - -- name: Create continuous query - command: > - sf-graph-render --influxdb-host "{{ influxdb_host }}" - --influxdb-password "{{ influxdb_admin_password }}" - update-cq - register: _update_cq - failed_when: _update_cq.rc not in [0, 4] - changed_when: _update_cq.rc == 4 diff --git a/ansible/roles/sf-influxdb/tasks/install.yml b/ansible/roles/sf-influxdb/tasks/install.yml deleted file mode 100644 index f911a0176..000000000 --- a/ansible/roles/sf-influxdb/tasks/install.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: Remove influxdb packages - yum: - name: "influxdb" - state: absent - -- name: Remove files - file: - path: "{{ item }}" - state: absent - loop: - - "/etc/yum.repos.d/{{ influxdb_repo_name }}.repo" - -- name: Pull image - include_role: - name: sf-container - tasks_from: install.yaml - loop: "{{ influxdb_components }}" diff --git a/ansible/roles/sf-influxdb/tasks/main.yml b/ansible/roles/sf-influxdb/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-influxdb/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-influxdb/tasks/setup.yml b/ansible/roles/sf-influxdb/tasks/setup.yml deleted file mode 100644 index 81e924f92..000000000 --- a/ansible/roles/sf-influxdb/tasks/setup.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -- include_tasks: "{{ sf_tasks_dir }}/create_user.yml" - vars: - user_name: "{{ owner }}" - -- name: Get {{ owner }} uid - command: id -u {{ owner }} - register: _influxdb_id - -- name: Get {{ owner }} gid - command: id -g {{ owner }} - register: _influxdb_gid - -- name: Setting {{ owner }} uid var - set_fact: - influxdb_uid: "{{ _influxdb_id.stdout }}" - influxdb_gid: "{{ _influxdb_gid.stdout }}" - -- name: Check container image version - include_role: - name: sf-container - tasks_from: check_version.yaml - loop: "{{ influxdb_components }}" - -- name: Create container directory - include_role: - name: sf-container - tasks_from: create_directory.yaml - loop: - - path: "{{ influxdb_config_dir }}" - recurse: "yes" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0750" - - path: "{{ influxdb_log_dir }}" - recurse: "yes" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0750" - - path: "{{ influxdb_data_dir }}" - recurse: "yes" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0750" - -- name: Apply selinux port labelling - seport: - ports: "{{ influxdb_port }}" - proto: "tcp" - setype: "http_port_t" - state: present - when: - - ansible_virtualization_type not in ["podman", "systemd-nspawn"] - - ansible_selinux.status == "enabled" - -- name: Setup config files - template: - src: "{{ item }}.j2" - dest: "{{ influxdb_config_dir }}/{{ item }}" - loop: - - influxdb.conf - -- name: Add crt and key files if needed - copy: - content: "{{ item.content }}" - dest: "{{ influxdb_config_dir }}/{{ item.dest }}" - mode: "{{ item.mode|default('0444') }}" - owner: "{{ owner }}" - setype: cert_t - seuser: system_u - serole: object_r - loop: - - {content: "{{ influxdb_crt}}", dest: "influxdb.crt"} - - {content: "{{ influxdb_key }}", dest: "influxdb.key", mode: '0400'} - no_log: "{{ show_hidden_logs }}" - -- name: Configure influxdb to use tls - ini_file: - dest: "{{ influxdb_config_dir }}/influxdb.conf" - section: '{{ item.section }}' - option: '{{ item.option }}' - value: '{{ item.value }}' - loop: - - {section: http, option: auth-enabled, value: 'true'} - - {section: http, option: https-enabled, value: 'true'} - - {section: http, option: https-certificate, value: '"{{ influxdb_config_dir }}/influxdb.crt"'} - - {section: http, option: https-private-key, value: '"{{ influxdb_config_dir }}/influxdb.key"'} - - {section: http, option: bind-address, value: '":{{ influxdb_port }}"'} - notify: restart influxdb - -- name: Create container - include_role: - name: sf-container - tasks_from: setup.yaml - loop: "{{ influxdb_components }}" - -- name: Restart influxdb when fqdn is updated to use new crt - systemd: - name: "{{ role_package }}" - state: restarted - when: update_fqdn - -- name: Start service - systemd: - name: "{{ role_package }}" - state: restarted - daemon_reload: true - enabled: true - -- include_tasks: influxdb_configuration.yml diff --git a/ansible/roles/sf-influxdb/tasks/upgrade.yml b/ansible/roles/sf-influxdb/tasks/upgrade.yml deleted file mode 100644 index f737ccaec..000000000 --- a/ansible/roles/sf-influxdb/tasks/upgrade.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Stop the service - service: - name: "{{ role_package }}" - state: stopped - -- name: Remove {{ role_package }} rpm package - yum: - name: "{{ role_package }}" - state: absent - -- name: Upgrade container - include_role: - name: sf-container - tasks_from: upgrade.yaml - loop: "{{ influxdb_components }}" diff --git a/ansible/roles/sf-influxdb/templates/influxdb.conf.j2 b/ansible/roles/sf-influxdb/templates/influxdb.conf.j2 deleted file mode 100644 index bf74773cb..000000000 --- a/ansible/roles/sf-influxdb/templates/influxdb.conf.j2 +++ /dev/null @@ -1,19 +0,0 @@ -[meta] -dir = "/var/lib/influxdb/meta" -[data] -dir = "/var/lib/influxdb/data" -wal-dir = "/var/lib/influxdb/wal" -series-id-set-cache-size = 100 -[coordinator] -[retention] -[shard-precreation] -[monitor] -[http] -[logging] -[subscriber] -[[graphite]] -[[collectd]] -[[opentsdb]] -[[udp]] -[continuous_queries] -[tls] \ No newline at end of file diff --git a/ansible/roles/sf-log-classify/defaults/main.yml b/ansible/roles/sf-log-classify/defaults/main.yml deleted file mode 100644 index 8e77b6e9b..000000000 --- a/ansible/roles/sf-log-classify/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -role_actions: - - install - - setup diff --git a/ansible/roles/sf-log-classify/meta/main.yml b/ansible/roles/sf-log-classify/meta/main.yml deleted file mode 100644 index d92779a36..000000000 --- a/ansible/roles/sf-log-classify/meta/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -galaxy_info: - description: Log-classify service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory diff --git a/ansible/roles/sf-log-classify/tasks/install.yml b/ansible/roles/sf-log-classify/tasks/install.yml deleted file mode 100644 index ed97d539c..000000000 --- a/ansible/roles/sf-log-classify/tasks/install.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/ansible/roles/sf-log-classify/tasks/main.yml b/ansible/roles/sf-log-classify/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-log-classify/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-log-classify/tasks/setup.yml b/ansible/roles/sf-log-classify/tasks/setup.yml deleted file mode 100644 index ed97d539c..000000000 --- a/ansible/roles/sf-log-classify/tasks/setup.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/ansible/roles/sf-opensearch-dashboards/defaults/main.yml b/ansible/roles/sf-opensearch-dashboards/defaults/main.yml deleted file mode 100644 index 1bfdd84a4..000000000 --- a/ansible/roles/sf-opensearch-dashboards/defaults/main.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -role_actions: - - install - - setup - - backup - - restore - - get_logs - - postconf - - disable - -role_package: opensearch-dashboards - -owner: opensearch-dashboards -group: opensearch-dashboards -container_uid: 1000 -container_gid: 1000 - -opensearch_dashboards_version: 2.4.0 -opensearch_dashboards_container_version: "{{ opensearch_dashboards_version }}-1" - -opensearch_dashboards_components: - - service: "opensearch-dashboards" - image: "quay.io/software-factory/opensearch-dashboards:{{ opensearch_dashboards_container_version }}" - params: >- - --net host - --user {{ container_uid }}:{{ container_gid }} - --uidmap 0:4000:999 - --uidmap {{ container_uid }}:{{ opensearch_dashboard_uid | default(1000) }}:1 - -v {{ opensearch_dashboards_certs_dir }}:{{ opensearch_dashboards_container_certs_dir }}:z - -v {{ elk_config_dir }}/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:z - {% if sf_opensearch_dashboards_additional_params is defined %} - {{ sf_opensearch_dashboards_additional_params }} - {% endif %} - -elk_config_dir: /etc/opensearch -elk_log_dir: /var/log/opensearch - -opensearch_dashboards_certs_dir: "{{ elk_config_dir }}/certs/opensearch-dashboards" - -opensearch_dashboards_container_dir: /usr/share/opensearch-dashboards -opensearch_dashboards_container_config_dir: "{{ opensearch_dashboards_container_dir }}/config" -opensearch_dashboards_container_certs_dir: "{{ opensearch_dashboards_container_config_dir }}/certificates" - -opensearch_http_port: 9200 -opensearch_host: opensearch.example.com -opensearch_internal_url: "https://{{ opensearch_host }}:{{ opensearch_http_port }}" -opensearch_readonly_user: kibana -opensearch_readonly_password: "CHANGE_ME" - -opensearch_dashboards_http_port: 5601 -opensearch_dashboards_host: opensearch_dashboards.example.com -opensearch_dashboards_internal_url: "http://{{ opensearch_dashboards_host }}:{{ opensearch_dashboards_http_port }}" -opensearch_dashboards_bundle_js_patch: "Readonly user: {{ opensearch_readonly_user }} password: {{ opensearch_readonly_password }}" - -keycloak_default_realm: SF -keycloak_kibana_client_secret: "CHANGE_ME" -keycloak_opensearch_dashboards_client_secret: "{{ keycloak_kibana_client_secret }}" - -# External opensearch configuration -external_opensearch_readonly_username: '' -external_opensearch_readonly_password: '' - -sf_cert_path: /var/lib/software-factory/bootstrap-data/certs diff --git a/ansible/roles/sf-opensearch-dashboards/files/kibana-backup.py b/ansible/roles/sf-opensearch-dashboards/files/kibana-backup.py deleted file mode 100755 index caf689e12..000000000 --- a/ansible/roles/sf-opensearch-dashboards/files/kibana-backup.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python3 - -# Original source: https://github.com/selivan/kibana-backup-simple -# Small improvements are done in fork: -# https://github.com/danpawlik/kibana-backup-simple - -# Kibana documentation: -# https://www.elastic.co/guide/en/kibana/current/saved-objects-api-export.html -# https://www.elastic.co/guide/en/kibana/current/saved-objects-api-import.html - -import datetime -import json -import sys -import time -import argparse -import requests -import os -import yaml - -# Error message from Kibana listing all possible saved objects types: -# \"type\" must be one of: -# [config, map, canvas-workpad, canvas-element, index-pattern, -# visualization, search, dashboard, url] -saved_objects_types = ('config', 'map', 'canvas-workpad', 'canvas-element', - 'index-pattern', 'visualization', 'search', 'dashboard', - 'url') - -to_remove_keys = ['updated_at', 'version', 'migrationVersion'] - - -def get_arguments(): - args_parser = argparse.ArgumentParser( - description='Backup and restore Kibana saved objects. ' - 'Writes backup to stdout and reads from stdin.') - args_parser.add_argument('action', - choices=['backup', 'restore', 'convert']) - args_parser.add_argument('--kibana-url', - default='http://127.0.0.1:5601', - help='URL to access Kibana API') - args_parser.add_argument('--space-id', - default='', - help='Kibana space id. If not set then the ' - 'default space is used.') - args_parser.add_argument('--user', default='', help='Kibana user') - args_parser.add_argument('--password', default='', help='Kibana password') - args_parser.add_argument('--backup-dir', - help='Dir where backups will be stored') - args_parser.add_argument('--file', help='File to restore or convert') - args_parser.add_argument('--restore-file', - help='DEPRECATED: File to restore or convert') - args_parser.add_argument('--resolve-conflicts', - default=True, - help='Resolve conflicts by removing index ' - 'id reference in backup file') - args_parser.add_argument('--insecure', - action='store_true', - help='Use that option to ignore if SSL cert ' - 'has been verified by root CA') - args_parser.add_argument('--extension', - default='ndjson', - help='Backup extension type') - args_parser.add_argument('--tenant', - help='Specify tenant for getting data.' - 'NOTE: if none is set, it will take Global') - args_parser.add_argument('--all-tenants', - action='store_true', - help='Bakup all objects in all ' - 'tenants. Works only with backup.' - 'NOTE: requires param: --elasticsearch-api-url') - args_parser.add_argument('--elasticsearch-api-url', - default='https://localhost:9200', - help='Require to get all tenants available in ' - 'elasticsearch') - return args_parser.parse_args() - - -def convert_to_yaml(text, remove_references): - # reparse text - text_lines = [] - try: - for line in text: - if isinstance(line, dict): - text_lines.append(line) - else: - text_lines.append(json.loads(line)) - except Exception as e: - print(e) - - if remove_references: - text_lines = remove_reference(text_lines) - return yaml.dump(text_lines) - - -def save_content_to_file(text, backup_file, extension, remove_references=True): - if isinstance(text, dict): - text = str(text) - if extension in ['yaml', 'yml']: - text = convert_to_yaml(text, remove_references) - elif extension in ['json', 'ndjson'] and isinstance(text, list): - text = " ".join(json.dumps(txt) + '\n' for txt in text) - with open(backup_file, 'a') as f: - f.write(text) - - -def parse_kibana_output(text): - new_text = [] - try: - text = [json.loads(text)] - except json.decoder.JSONDecodeError: - for text_obj in text.rsplit('\n'): - n_text = json.loads(text_obj) - new_text.append(n_text) - return new_text if new_text else text - - -def check_if_empty(text): - text = json.loads(text) - if 'exportedCount' in text and text['exportedCount'] == 0: - return True - - -def remove_obj_keys(ref): - for k in to_remove_keys: - ref.pop(k, None) - return ref - - -def remove_reference(text): - new_text = [] - new_references = [] - for text_obj in text: - for ref in text_obj['references']: - if (not ref.get('id').startswith('AX') - and len(ref.get('id')) != 20): - new_references.append(remove_obj_keys(ref)) - text_obj['references'] = new_references - new_text.append(text_obj) - return new_text if new_text else text - - -def make_request(url, user, password, text, tenant, insecure=False, - retry=True): - r = None - headers = {'kbn-xsrf': 'reporting', 'osd-xsrf': 'true'} - if tenant: - headers['securitytenant'] = tenant - - try: - r = requests.post(url, - auth=(user, password), - headers=headers, - files={'file': ('backup.ndjson', text)}, - timeout=10, - verify=insecure) - except requests.exceptions.ReadTimeout: - if not retry: - print("Importing failed. Retrying...") - time.sleep(10) - make_request(url, user, password, text, tenant, insecure) - - if r and "Please enter your credentials" in r.text: - print("Please provide correct username and password") - sys.exit(1) - - return r - - -def _get_file_content(backup_file): - if (backup_file.endswith('yml') or backup_file.endswith('yaml')): - with open(backup_file) as f: - text = yaml.safe_load(f) - else: - with open(backup_file) as f: - text = f.readlines() - return text - - -def backup(kibana_url, space_id, user, password, backup_dir, insecure, - tenant, extension='ndjson'): - """Return string with newline-delimitered json containing - Kibana saved objects""" - saved_objects = {} - if not backup_dir: - backup_dir = os.path.dirname(os.path.realpath(__file__)) - - # Set the same time for all backups if previous exists - b_time = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M") - - if len(space_id): - url = kibana_url + '/s/' + space_id + '/api/saved_objects/_export' - else: - url = kibana_url + '/api/saved_objects/_export' - for obj_type in saved_objects_types: - print("Working on %s" % obj_type) - - # osd-xsrf header is required by opensearch - # https://opensearch.org/docs/latest/troubleshoot/index/ - headers = {'Content-Type': 'application/json', - 'kbn-xsrf': 'reporting', - 'osd-xsrf': 'true'} - if tenant: - headers['securitytenant'] = tenant - - r = requests.post(url, - auth=(user, password), - headers=headers, - data='{ "type": "' + obj_type + - '","excludeExportDetails": true}', - verify=insecure) - - if r.status_code == 400: - # Print warning on missing object, but continue - print("Can not backup object %s" % obj_type) - continue - else: - r.raise_for_status() - - if not r.text: - continue - - if tenant: - backup_file = "%s/%s-%s.%s" % (backup_dir, obj_type, tenant, - extension) - else: - backup_file = "%s/%s.%s" % (backup_dir, obj_type, extension) - - if os.path.exists(backup_file): - backup_file = "%s-%s" % (backup_file, b_time) - - text = parse_kibana_output(r.text) - saved_objects[obj_type] = text - save_content_to_file(text, backup_file, extension) - - if tenant: - backup_file = "%s/backup-%s.%s" % (backup_dir, tenant, extension) - else: - backup_file = "%s/backup.%s" % (backup_dir, extension) - if os.path.exists(backup_file): - backup_file = "%s-%s" % (backup_file, b_time) - - for kib_obj in saved_objects.values(): - save_content_to_file(kib_obj, backup_file, extension, False) - - -def restore(kibana_url, space_id, user, password, text, resolve_conflicts, - insecure, tenant): - """Restore given newline-delimitered json containing - saved objects to Kibana""" - - if len(space_id): - url = (kibana_url + '/s/' + space_id + - '/api/saved_objects/_import?overwrite=true') - else: - url = kibana_url + '/api/saved_objects/_import?overwrite=true' - - if not isinstance(text, list): - text = [text] - - for kib_obj in text: - print("Working on %s" % kib_obj) - - if not isinstance(kib_obj, dict): - # Ensure that the kib_obj is one-time converted json object - kib_obj = json.dumps(json.loads(kib_obj)) - else: - kib_obj = json.dumps(kib_obj) - - if check_if_empty(kib_obj): - print("Spotted empty object. Continue...") - continue - - r = make_request(url, user, password, kib_obj, tenant, insecure) - - if r.status_code == 401: - print("Unauthorized. Please provide user and password") - - try: - response_error = json.loads(r.text) - if response_error.get('errors'): - print("\n\nSome problem on restoring %s: %s\n\n" % - (kib_obj, response_error['errors'])) - except Exception as e: - print("Kibana restore requests objects does not look correct:" - " %s" % e) - - if not r: - print("Can not import %s into Kibana" % kib_obj) - continue - - response_text = json.loads(r.text) - if not response_text['success'] and resolve_conflicts: - text = remove_reference(kib_obj) - r = make_request(url, user, password, text, tenant, insecure) - - print(r.status_code, r.reason, '\n', r.text) - r.raise_for_status() # Raises stored HTTPError, if one occurred. - - -def convert(text, extension, convert_file): - convert_file = "%s-converted.%s" % (convert_file, extension) - save_content_to_file(text, convert_file, extension, False) - - -def get_all_tenants(elasticsearch_api_url, user, password, insecure): - url = "%s/_opendistro/_security/api/tenants/" % elasticsearch_api_url - r = requests.get(url, auth=(user, password), verify=insecure) - if r.status_code != 200: - r.raise_for_status() - sys.exit(1) - return list(json.loads(r.text)) - - -if __name__ == '__main__': - args = get_arguments() - kibana_url = args.kibana_url - - if args.file and args.restore_file: - print("Can not set both params: file and restore-file. Exit") - sys.exit(1) - - if (not args.kibana_url.startswith('http') - and not args.kibana_url.startswith('https')): - kibana_url = "https://%s" % args.kibana_url - - if args.action == 'backup': - if args.all_tenants and args.tenant: - print("Can not use --all-tenants with --tenant option") - sys.exit(1) - - if args.all_tenants: - if not args.elasticsearch_api_url: - print('Please provide --elasticsearch-api-url to list all' - ' tenants available in Elasticsearch.') - sys.exit(1) - all_tenants = get_all_tenants(args.elasticsearch_api_url, - args.user, args.password, - args.insecure) - - for tenant in all_tenants: - backup(kibana_url, args.space_id, args.user, args.password, - args.backup_dir, args.insecure, tenant, args.extension) - else: - backup(kibana_url, args.space_id, args.user, args.password, - args.backup_dir, args.insecure, args.tenant, args.extension) - - elif args.action == 'restore': - restore_file = args.file if args.file else args.restore_file - if restore_file: - text = _get_file_content(restore_file) - else: - text = ''.join(sys.stdin.readlines()) - - restore(kibana_url, args.space_id, args.user, args.password, text, - args.resolve_conflicts, args.insecure, args.tenant) - - elif args.action == 'convert': - if args.file: - text = _get_file_content(args.file) - else: - text = ''.join(sys.stdin.readlines()) - if not text: - print("Can not continue. Did not provide --file param or stdin") - sys.exit(1) - - convert(text, args.extension, args.file) diff --git a/ansible/roles/sf-opensearch-dashboards/handlers/main.yml b/ansible/roles/sf-opensearch-dashboards/handlers/main.yml deleted file mode 100644 index fc7d7f06d..000000000 --- a/ansible/roles/sf-opensearch-dashboards/handlers/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: restart opensearch dashboards - systemd: - name: opensearch-dashboards - state: restarted - daemon_reload: true - -- name: ensure started opensearch dashboards - systemd: - name: opensearch-dashboards - state: started - enabled: "yes" - daemon_reload: true - -- name: wait for Opensearch dashboards service - uri: - url: "https://{{ fqdn }}/analytics/dashboards" - method: GET - validate_certs: false - status_code: "401" - return_content: true - register: _opensearch_dashboards_status - until: "'Unauthorized' in _opensearch_dashboards_status.content" - retries: 30 - delay: 10 diff --git a/ansible/roles/sf-opensearch-dashboards/meta/main.yml b/ansible/roles/sf-opensearch-dashboards/meta/main.yml deleted file mode 100644 index 21bb4c04d..000000000 --- a/ansible/roles/sf-opensearch-dashboards/meta/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -galaxy_info: - description: OpenSearch Dashboards service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory diff --git a/ansible/roles/sf-opensearch-dashboards/meta/sfconfig.py b/ansible/roles/sf-opensearch-dashboards/meta/sfconfig.py deleted file mode 100644 index 507e0dfd1..000000000 --- a/ansible/roles/sf-opensearch-dashboards/meta/sfconfig.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sfconfig.components import Component - - -class Kibana(Component): - def configure(self, args, host): - - if 'kibana' in args.sfconfig: - args.glue['readonly_user_autologin'] = \ - args.sfconfig.get("kibana", {}).get('readonly_user_autologin', - 'Basic') - elif 'opensearch_dashboards' in args.sfconfig: - args.glue['readonly_user_autologin'] = \ - args.sfconfig.get("opensearch_dashboards", {}).get( - 'readonly_user_autologin', 'Basic') - - self.get_or_generate_cert(args, "opensearch-dashboards", - host["hostname"]) diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/backup.yml b/ansible/roles/sf-opensearch-dashboards/tasks/backup.yml deleted file mode 100644 index ca6bbf7dd..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/backup.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Check if Kibana service is alive - wait_for: - host: "{{ opensearch_dashboards_host }}" - port: 5601 - timeout: 1 - register: kibana_status - ignore_errors: true - -- name: Fail when Kibana is not available - fail: - msg: "Can not connect to Kibana to perform backup" - when: kibana_status is not successful - -- name: Create backup of all Kibana objects - shell: | - /usr/local/bin/kibana-backup.py \ - --user kibanaserver \ - --password "{{ opensearch['kibanaserver_password'] | default(opensearch_kibanaserver_password) }}" \ - --kibana-url "https://{{ opensearch_dashboards_host }}/analytics" \ - --backup-dir {{ backup_dest }} \ - backup diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/disable.yml b/ansible/roles/sf-opensearch-dashboards/tasks/disable.yml deleted file mode 100644 index dac0062f9..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/disable.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Disable services - service: - name: opensearch-dashboards - state: stopped - enabled: false - ignore_errors: true - -- name: Delete containers - include_role: - name: sf-container - tasks_from: delete.yaml - loop: "{{ opensearch_dashboards_components }}" - when: erase == True diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/get_logs.yml b/ansible/roles/sf-opensearch-dashboards/tasks/get_logs.yml deleted file mode 100644 index cc3a908df..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/get_logs.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Dump service logs - shell: journalctl -u opensearch-dashboards > /var/log/opensearch/opensearch-dashboards.log - ignore_errors: true - -- name: Fetch service logs - synchronize: - mode: pull - src: "{{ elk_log_dir }}/" - dest: "{{ log_dest }}/log" - ignore_errors: true diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/install.yml b/ansible/roles/sf-opensearch-dashboards/tasks/install.yml deleted file mode 100644 index 76492ef6a..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/install.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- include_tasks: "{{ sf_tasks_dir }}/create_user.yml" - vars: - user_name: opensearch-dashboards - -- name: Pull image - include_role: - name: sf-container - tasks_from: install.yaml - loop: "{{ opensearch_dashboards_components }}" diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/main.yml b/ansible/roles/sf-opensearch-dashboards/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/postconf.yml b/ansible/roles/sf-opensearch-dashboards/tasks/postconf.yml deleted file mode 100644 index a5d819131..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/postconf.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- name: Ensure that service is up and running - service: - name: opensearch-dashboards - state: started - notify: - - wait for Opensearch dashboards service - -- meta: flush_handlers - -- name: Validate if jwt token is valid - block: - - include_vars: - file: /var/lib/software-factory/bootstrap-data/secrets.yaml - name: opensearch_kibanauser_token - - - name: Making query with jwt header - uri: - url: "https://{{ fqdn }}/analytics_autologin/app/kibana_overview" - method: GET - headers: - Authorization: "Bearer {{ opensearch_kibanauser_token }}" - status_code: "200" - validate_certs: true - when: readonly_user_autologin | lower == 'jwt' - -- name: Validate if basic auth is valid - block: - - include_vars: - file: /var/lib/software-factory/bootstrap-data/secrets.yaml - name: opensearch_kibanauser_basicauth - - - name: Making query with basicauth header - uri: - url: "https://{{ fqdn }}/analytics_autologin/app/kibana_overview" - method: GET - headers: - Authorization: "Basic {{ opensearch_kibanauser_basicauth }}" - status_code: "200" - validate_certs: true - when: readonly_user_autologin | lower == 'basic' diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/restore.yml b/ansible/roles/sf-opensearch-dashboards/tasks/restore.yml deleted file mode 100644 index c4acdf5a1..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/restore.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# NOTE(dpawlik) uncomment that, when sf-config will have a feature, -# that restore backup will be after service is running. - -#- name: Check if Kibana service is alive -# wait_for: -# host: "{{ opensearch_dashboards_host }}" -# port: 5601 -# timeout: 1 -# register: kibana_status -# ignore_errors: true -# -#- name: Fail when Kibana is not available -# fail: -# msg: "Can not connect to Kibana to restore backup" -# when: kibana_status is not successful -# -#- name: Restore objects from backup -# command: | -# /usr/local/bin/kibana-backup.py --kibana-url "{{ opensearch_dashboards_internal_url }}" --restore-file {{ backup_src }}/backup.ndjson restore diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/setup.yml b/ansible/roles/sf-opensearch-dashboards/tasks/setup.yml deleted file mode 100644 index 5797cea2f..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/setup.yml +++ /dev/null @@ -1,96 +0,0 @@ ---- -- name: Check container image version - include_role: - name: sf-container - tasks_from: check_version.yaml - loop: "{{ opensearch_dashboards_components }}" - -- name: Create cert dir - file: - path: "{{ opensearch_dashboards_certs_dir }}" - owner: "{{ owner }}" - group: "{{ group }}" - state: directory - -- name: Copy certs - copy: - src: "{{ sf_cert_path }}/{{ item }}" - dest: "{{ opensearch_dashboards_certs_dir }}/{{ item }}" - mode: "0644" - owner: "{{ owner }}" - group: "{{ group }}" - loop: - - opensearch-dashboards.crt - - opensearch-dashboards.key - - localCA.pem - -- name: Generate Opensearch dashboards welcome page text - internal host - set_fact: - opensearch_dashboards_login_screen: "Readonly user: {{ opensearch_readonly_user }} password: {{ opensearch_readonly_password }}" - -- name: Generate Opensearch dashboards welcome page text - external host - set_fact: - opensearch_dashboards_login_screen: "Readonly user: {{ external_opensearch_readonly_username }} password: {{ external_opensearch_readonly_password }}" - when: external_opensearch_readonly_username != '' and external_opensearch_readonly_password != '' - -- name: Configure Opensearch Dashboards - template: - src: opensearch-dashboards.yml.j2 - dest: /etc/opensearch/opensearch_dashboards.yml - owner: "{{ owner }}" - group: "{{ group }}" - notify: - - restart opensearch dashboards - - ensure started opensearch dashboards - -- name: Get opensearch-dashboards uid - command: id -u opensearch-dashboards - register: _opensearch_dashboards_id - -- name: Expose opensearch-dashboards uid - set_fact: - opensearch_dashboards_uid: "{{ _opensearch_dashboards_id.stdout }}" - -- name: Create container - Opensearch dashboards - include_role: - name: sf-container - tasks_from: setup.yaml - loop: "{{ opensearch_dashboards_components }}" - -- name: Send backup and restore script - copy: - src: kibana-backup.py - dest: /usr/local/bin/kibana-backup.py - mode: '0755' - -# NOTE(dpawlik) Remove that when sf-config will have a feature, that -# restore backup will be done after starting the service. -- name: Restore Opensearch Dashboards objects that was available before moving to Opendistro - block: - - name: Set backup src var - set_fact: - backup_src: "/var/lib/software-factory/backup/opensearch_dashboards" - - - name: Check if Opensearch Dashboards service is alive - wait_for: - host: "{{ opensearch_dashboards_host }}" - port: 5601 - timeout: 300 - delay: 20 - - - name: Check it there is a backup.ndjson file - stat: - path: "{{ backup_src }}/backup.ndjson" - register: kibana_backup_file - - - name: Restore backup, when file exists - block: - - name: Restore objects from backup - command: | - /usr/local/bin/kibana-backup.py --kibana-url "{{ opensearch_dashboards_internal_url }}" --restore-file {{ backup_src }}/backup.ndjson restore - - name: Rename backup.ndjson to other name - command: | - mv {{ backup_src }}/backup.ndjson {{ backup_src }}/backup.ndjson-restored - when: kibana_backup_file.stat.exists - - when: sf_previous_version < 3.6 diff --git a/ansible/roles/sf-opensearch-dashboards/tasks/upgrade.yml b/ansible/roles/sf-opensearch-dashboards/tasks/upgrade.yml deleted file mode 100644 index ca89dc582..000000000 --- a/ansible/roles/sf-opensearch-dashboards/tasks/upgrade.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Get available service facts - service_facts: - -- name: Stop Opensearch dashboards service - systemd: - name: opensearch-dashboards - state: stopped - when: "'opensearch-dashboards.service' in services" - -- name: Upgrade container - include_role: - name: sf-container - tasks_from: upgrade.yaml - loop: "{{ opensearch_dashboards_components }}" diff --git a/ansible/roles/sf-opensearch-dashboards/templates/opensearch-dashboards.yml.j2 b/ansible/roles/sf-opensearch-dashboards/templates/opensearch-dashboards.yml.j2 deleted file mode 100644 index 6104d8f2e..000000000 --- a/ansible/roles/sf-opensearch-dashboards/templates/opensearch-dashboards.yml.j2 +++ /dev/null @@ -1,39 +0,0 @@ ---- -opensearch.hosts: ["{{ opensearch_internal_url }}"] - -server.basePath: "/analytics" -server.host: 0.0.0.0 -server.ssl.enabled: true -server.ssl.certificate: {{ opensearch_dashboards_container_certs_dir }}/opensearch-dashboards.crt -server.ssl.key: {{ opensearch_dashboards_container_certs_dir }}/opensearch-dashboards.key - -opensearch.ssl.verificationMode: full -opensearch.username: kibanaserver -opensearch.password: {{ opensearch_kibanaserver_password }} -opensearch.ssl.certificateAuthorities: ["{{ opensearch_dashboards_container_certs_dir }}/localCA.pem"] - -opensearch_security.multitenancy.tenants.enable_private: false -opensearch_security.multitenancy.enabled: false -opensearch_security.multitenancy.tenants.preferred: [ "Global" ] -opensearch_security.multitenancy.enable_filter: false -opensearch_security.multitenancy.tenants.enable_global: true -opensearch_security.basicauth.login.title: "The Software Factory Elasticsearch" -{% if opensearch_dashboards_login_screen is defined and opensearch_dashboards_login_screen != '' %} -opensearch_security.basicauth.login.subtitle: "{{ opensearch_dashboards_login_screen }}" -{% else %} -opensearch_security.basicauth.login.subtitle: "Username: kibana Password: kibana" -{% endif %} - -# additional settings that were not listened in configure-kibana, -# but are required. -opensearch.requestHeadersWhitelist: ["securitytenant","Authorization"] -opensearch_security.readonly_mode.roles: ["kibana_read_only"] -opensearch_security.cookie.secure: false -map.includeElasticMapsService: false - -opensearch_security.auth.type: "openid" -opensearch_security.openid.client_id: opensearch -opensearch_security.openid.client_secret: {{ keycloak_opensearch_client_secret }} -opensearch_security.openid.connect_url: https://{{ fqdn }}/auth/realms/{{ keycloak_default_realm }}/.well-known/openid-configuration -opensearch_security.openid.base_redirect_url: https://{{ fqdn }}/analytics/ -opensearch_security.openid.root_ca: {{ opensearch_dashboards_container_certs_dir }}/localCA.pem diff --git a/ansible/roles/sf-opensearch/defaults/main.yml b/ansible/roles/sf-opensearch/defaults/main.yml deleted file mode 100644 index e74a8ea67..000000000 --- a/ansible/roles/sf-opensearch/defaults/main.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -role_actions: - - install - - setup - - get_logs - - postconf - - disable - -role_package: opensearch - -owner: opensearch -group: opensearch -container_uid: 1000 -container_gid: 1000 - -opensearch_version: 2.4.0 -opensearch_container_version: "{{ opensearch_version }}-1" - -opensearch_components: - - service: "opensearch" - image: "quay.io/software-factory/opensearch:{{ opensearch_container_version }}" - params: >- - --net host - --user {{ container_uid }}:{{ container_gid }} - --uidmap 0:4000:999 - --uidmap {{ container_uid }}:{{ opensearch_uid | default(1000) }}:1 - -e DISABLE_INSTALL_DEMO_CONFIG=true - -e "OPENSEARCH_JAVA_OPTS=-Xms{{ opensearch_minimum_heap_size }} -Xmx{{ opensearch_maximum_heap_size }}" - -v {{ elk_data_dir }}:{{ opensearch_container_dir }}/data:z - -v {{ elk_log_dir }}:{{ elk_log_dir }}:z - -v {{ elk_log_dir }}/others:/usr/share/opensearch/logs:z - -v {{ elk_stack_certs }}:{{ opensearch_container_certs_dir }}:z - -v {{ elk_config_dir }}/opensearch.yml:{{ opensearch_container_config_dir }}/opensearch.yml:z - -v {{ elk_config_dir }}/log4j2.properties:{{ opensearch_container_config_dir }}/log4j2.properties:z - -v {{ elk_config_dir }}/config.yml:{{ opensearch_container_securityconfig_dir }}/config.yml:z - -v {{ elk_config_dir }}/internal_users.yml:{{ opensearch_container_securityconfig_dir }}/internal_users.yml:z - -v {{ elk_config_dir }}/roles_mapping.yml:{{ opensearch_container_securityconfig_dir }}/roles_mapping.yml:z - -v {{ elk_config_dir }}/tenants.yml:{{ opensearch_container_securityconfig_dir }}/tenants.yml:z - -v {{ elk_config_dir }}/roles.yml:{{ opensearch_container_securityconfig_dir }}/roles.yml:z - -v {{ elk_config_dir }}/action_groups.yml:{{ opensearch_container_securityconfig_dir }}/action_groups.yml:z - -v {{ elk_config_dir }}/nodes_dn.yml:{{ opensearch_container_securityconfig_dir }}/nodes_dn.yml:z - -v {{ elk_config_dir }}/whitelist.yml:{{ opensearch_container_securityconfig_dir }}/whitelist.yml:z - {% if sf_opensearch_additional_params is defined %} - {{ sf_opensearch_additional_params }} - {% endif %} - -opensearch_container_dir: /usr/share/opensearch -opensearch_container_config_dir: "{{ opensearch_container_dir }}/config" -opensearch_container_certs_dir: "{{ opensearch_container_config_dir }}/certificates" -opensearch_container_securityconfig_dir: "{{ opensearch_container_config_dir }}/opensearch-security" - -elk_config_dir: /etc/opensearch -elk_data_dir: /var/lib/elasticsearch -elk_log_dir: /var/log/opensearch -elk_stack_certs: "{{ elk_config_dir }}/certs/opensearch" -old_elk_config_dir: /etc/elasticsearch -old_elk_stack_certs: "{{ old_elk_config_dir }}/certs" - -opensearch_password: "CHANGE_ME" -opensearch_kibanaserver_password: "CHANGE_ME" -opensearch_logstash_password: "CHANGE_ME" -opensearch_zuul_password: "CHANGE_ME" -opensearch_readonly_user: kibana -opensearch_readonly_password: "CHANGE_ME" - -opensearch_http_port: 9200 -opensearch_host: opensearch.example.com -opensearch_internal_url: "https://{{ opensearch_host }}:{{ opensearch_http_port }}" - -opensearch_maximum_heap_size: 512m -opensearch_minimum_heap_size: 128m -opensearch_replicas: 0 - -enable_performance_analyzer: false - -keycloak_default_realm: SF diff --git a/ansible/roles/sf-opensearch/files/log4j2.properties b/ansible/roles/sf-opensearch/files/log4j2.properties deleted file mode 100644 index bb27aaf2e..000000000 --- a/ansible/roles/sf-opensearch/files/log4j2.properties +++ /dev/null @@ -1,234 +0,0 @@ -# -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# - -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -######## Server JSON ############################ -appender.rolling.type = RollingFile -appender.rolling.name = rolling -appender.rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_server.json -appender.rolling.filePermissions = rw-r----- -appender.rolling.layout.type = OpenSearchJsonLayout -appender.rolling.layout.type_name = server - -appender.rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz -appender.rolling.policies.type = Policies -appender.rolling.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval = 1 -appender.rolling.policies.time.modulate = true -appender.rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling.policies.size.size = 128MB -appender.rolling.strategy.type = DefaultRolloverStrategy -appender.rolling.strategy.fileIndex = nomax -appender.rolling.strategy.action.type = Delete -appender.rolling.strategy.action.basepath = ${sys:opensearch.logs.base_path} -appender.rolling.strategy.action.condition.type = IfFileName -appender.rolling.strategy.action.condition.glob = ${sys:opensearch.logs.cluster_name}-* -appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize -appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB -################################################ -######## Server - old style pattern ########### -appender.rolling_old.type = RollingFile -appender.rolling_old.name = rolling_old -appender.rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}.log -appender.rolling_old.filePermissions = rw-r----- -appender.rolling_old.layout.type = PatternLayout -appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz -appender.rolling_old.policies.type = Policies -appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy -appender.rolling_old.policies.time.interval = 1 -appender.rolling_old.policies.time.modulate = true -appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling_old.policies.size.size = 128MB -appender.rolling_old.strategy.type = DefaultRolloverStrategy -appender.rolling_old.strategy.fileIndex = nomax -appender.rolling_old.strategy.action.type = Delete -appender.rolling_old.strategy.action.basepath = ${sys:opensearch.logs.base_path} -appender.rolling_old.strategy.action.condition.type = IfFileName -appender.rolling_old.strategy.action.condition.glob = ${sys:opensearch.logs.cluster_name}-* -appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize -appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB -################################################ - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console -rootLogger.appenderRef.rolling.ref = rolling -rootLogger.appenderRef.rolling_old.ref = rolling_old - -######## Deprecation JSON ####################### -appender.deprecation_rolling.type = RollingFile -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation.json -appender.deprecation_rolling.filePermissions = rw-r----- -appender.deprecation_rolling.layout.type = OpenSearchJsonLayout -appender.deprecation_rolling.layout.type_name = deprecation -appender.deprecation_rolling.layout.opensearchmessagefields=x-opaque-id -appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter - -appender.deprecation_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation-%i.json.gz -appender.deprecation_rolling.policies.type = Policies -appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling.policies.size.size = 1GB -appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling.strategy.max = 4 - -appender.header_warning.type = HeaderWarningAppender -appender.header_warning.name = header_warning -################################################# -######## Deprecation - old style pattern ####### -appender.deprecation_rolling_old.type = RollingFile -appender.deprecation_rolling_old.name = deprecation_rolling_old -appender.deprecation_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation.log -appender.deprecation_rolling_old.filePermissions = rw-r----- -appender.deprecation_rolling_old.layout.type = PatternLayout -appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.deprecation_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _deprecation-%i.log.gz -appender.deprecation_rolling_old.policies.type = Policies -appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling_old.policies.size.size = 1GB -appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling_old.strategy.max = 4 -################################################# -logger.deprecation.name = org.opensearch.deprecation -logger.deprecation.level = deprecation -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old -logger.deprecation.appenderRef.header_warning.ref = header_warning -logger.deprecation.additivity = false - -######## Search slowlog JSON #################### -appender.index_search_slowlog_rolling.type = RollingFile -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\ - .cluster_name}_index_search_slowlog.json -appender.index_search_slowlog_rolling.filePermissions = rw-r----- -appender.index_search_slowlog_rolling.layout.type = OpenSearchJsonLayout -appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog -appender.index_search_slowlog_rolling.layout.opensearchmessagefields=message,took,took_millis,total_hits,types,stats,search_type,total_shards,source,id - -appender.index_search_slowlog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\ - .cluster_name}_index_search_slowlog-%i.json.gz -appender.index_search_slowlog_rolling.policies.type = Policies -appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.index_search_slowlog_rolling.policies.size.size = 1GB -appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy -appender.index_search_slowlog_rolling.strategy.max = 4 -################################################# -######## Search slowlog - old style pattern #### -appender.index_search_slowlog_rolling_old.type = RollingFile -appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old -appender.index_search_slowlog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _index_search_slowlog.log -appender.index_search_slowlog_rolling_old.filePermissions = rw-r----- -appender.index_search_slowlog_rolling_old.layout.type = PatternLayout -appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.index_search_slowlog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _index_search_slowlog-%i.log.gz -appender.index_search_slowlog_rolling_old.policies.type = Policies -appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.index_search_slowlog_rolling_old.policies.size.size = 1GB -appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.index_search_slowlog_rolling_old.strategy.max = 4 -################################################# -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = trace -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old -logger.index_search_slowlog_rolling.additivity = false - -######## Indexing slowlog JSON ################## -appender.index_indexing_slowlog_rolling.type = RollingFile -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _index_indexing_slowlog.json -appender.index_indexing_slowlog_rolling.filePermissions = rw-r----- -appender.index_indexing_slowlog_rolling.layout.type = OpenSearchJsonLayout -appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog -appender.index_indexing_slowlog_rolling.layout.opensearchmessagefields=message,took,took_millis,doc_type,id,routing,source - -appender.index_indexing_slowlog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _index_indexing_slowlog-%i.json.gz -appender.index_indexing_slowlog_rolling.policies.type = Policies -appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling.policies.size.size = 1GB -appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy -appender.index_indexing_slowlog_rolling.strategy.max = 4 -################################################# -######## Indexing slowlog - old style pattern ## -appender.index_indexing_slowlog_rolling_old.type = RollingFile -appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old -appender.index_indexing_slowlog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling_old.filePermissions = rw-r----- -appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ - _index_indexing_slowlog-%i.log.gz -appender.index_indexing_slowlog_rolling_old.policies.type = Policies -appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB -appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.index_indexing_slowlog_rolling_old.strategy.max = 4 -################################################# - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = trace -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old -logger.index_indexing_slowlog.additivity = false - -######## Task details log JSON #################### -appender.task_detailslog_rolling.type = RollingFile -appender.task_detailslog_rolling.name = task_detailslog_rolling -appender.task_detailslog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog.json -appender.task_detailslog_rolling.filePermissions = rw-r----- -appender.task_detailslog_rolling.layout.type = OpenSearchJsonLayout -appender.task_detailslog_rolling.layout.type_name = task_detailslog -appender.task_detailslog_rolling.layout.opensearchmessagefields=taskId,type,action,description,start_time_millis,resource_stats,metadata - -appender.task_detailslog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog-%i.json.gz -appender.task_detailslog_rolling.policies.type = Policies -appender.task_detailslog_rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.task_detailslog_rolling.policies.size.size = 1GB -appender.task_detailslog_rolling.strategy.type = DefaultRolloverStrategy -appender.task_detailslog_rolling.strategy.max = 4 -################################################# -######## Task details log - old style pattern #### -appender.task_detailslog_rolling_old.type = RollingFile -appender.task_detailslog_rolling_old.name = task_detailslog_rolling_old -appender.task_detailslog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog.log -appender.task_detailslog_rolling_old.filePermissions = rw-r----- -appender.task_detailslog_rolling_old.layout.type = PatternLayout -appender.task_detailslog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.task_detailslog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog-%i.log.gz -appender.task_detailslog_rolling_old.policies.type = Policies -appender.task_detailslog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.task_detailslog_rolling_old.policies.size.size = 1GB -appender.task_detailslog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.task_detailslog_rolling_old.strategy.max = 4 -################################################# -logger.task_detailslog_rolling.name = task.detailslog -logger.task_detailslog_rolling.level = trace -logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling.ref = task_detailslog_rolling -logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling_old.ref = task_detailslog_rolling_old -logger.task_detailslog_rolling.additivity = false diff --git a/ansible/roles/sf-opensearch/handlers/main.yml b/ansible/roles/sf-opensearch/handlers/main.yml deleted file mode 100644 index e46bc7678..000000000 --- a/ansible/roles/sf-opensearch/handlers/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: restart opensearch - systemd: - name: opensearch - state: restarted - daemon_reload: true - -- name: ensure started opensearch - systemd: - name: opensearch - state: started - enabled: "yes" - -- name: wait for opensearch service - uri: - url: "{{ opensearch_internal_url }}" - method: GET - validate_certs: false - status_code: "401" - register: _opensearch_service_status - until: "(_opensearch_service_status is successful) and (_opensearch_service_status.status == 401)" - retries: 30 - delay: 10 - -- name: reconfigure security plugin in containers - command: /usr/local/bin/reconfigure-opensearch.sh diff --git a/ansible/roles/sf-opensearch/meta/main.yml b/ansible/roles/sf-opensearch/meta/main.yml deleted file mode 100644 index e70ec3e24..000000000 --- a/ansible/roles/sf-opensearch/meta/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -galaxy_info: - description: OpenSearch service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory diff --git a/ansible/roles/sf-opensearch/meta/sfconfig.py b/ansible/roles/sf-opensearch/meta/sfconfig.py deleted file mode 100644 index c59492549..000000000 --- a/ansible/roles/sf-opensearch/meta/sfconfig.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sfconfig.components import Component - - -class OpensSearch(Component): - def configure(self, args, host): - if 'minimum_heap_size' in args.sfconfig['opensearch']: - args.glue['opensearch_minimum_heap_size'] = args.sfconfig[ - 'opensearch']['minimum_heap_size'] - if 'maximum_heap_size' in args.sfconfig['opensearch']: - args.glue['opensearch_maximum_heap_size'] = args.sfconfig[ - 'opensearch']['maximum_heap_size'] - if 'replicas' in args.sfconfig['opensearch']: - args.glue['opensearch_replicas'] = args.sfconfig[ - 'opensearch']['replicas'] - - self.get_or_generate_cert(args, "opensearch-admin", - host["hostname"]) - - # The internal Elasticsearch connection should not be included in - # sfconfig. Add other connections that will be used by zuul. - args.glue["opensearch_connections"].append({ - 'name': "opensearch", - 'username': 'zuul', - 'password': args.secrets['opensearch_zuul_password'], - 'host': args.glue["opensearch_host"], - 'port': args.defaults["opensearch_http_port"], - }) - - args.glue["opensearch_connections"] = [dict(t) for t in { - tuple(d.items()) for d in args.glue["opensearch_connections"]}] - - args.glue['readonly_user_autologin'] = \ - args.sfconfig.get("opensearch_dashboards", {}).get( - 'readonly_user_autologin', 'Basic') diff --git a/ansible/roles/sf-opensearch/tasks/create_ism.yml b/ansible/roles/sf-opensearch/tasks/create_ism.yml deleted file mode 100644 index 3c001a7e6..000000000 --- a/ansible/roles/sf-opensearch/tasks/create_ism.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# NOTE: The ISM policy is automatically attached to the index, that -# matches the template pattern. For example: -# ism_template: {"intex_patterns": ["logstash-*"]} -# any index that will be created with name "logstash-" will be automatically -# attached to the policy. -- name: Create a ISM policy - uri: - url: "{{ opensearch_internal_url }}/_plugins/_ism/add/{{ item.index_name | default('universal') }}" - client_cert: "{{ elk_stack_certs }}/opensearch-admin.crt" - client_key: "{{ elk_stack_certs }}/opensearch-admin.key" - method: POST - body_format: json - body: {"policy_id": "{{ item.policy_name }}"} - status_code: 200, 409 - validate_certs: true - -- name: Add template to policy - uri: - url: "{{ opensearch_internal_url }}/_plugins/_ism/policies/{{ item.policy_name }}" - client_cert: "{{ elk_stack_certs }}/opensearch-admin.crt" - client_key: "{{ elk_stack_certs }}/opensearch-admin.key" - method: PUT - body_format: json - body: "{{ lookup('template','./ism.json.j2') }}" - status_code: 201, 409 - validate_certs: true diff --git a/ansible/roles/sf-opensearch/tasks/create_users.yml b/ansible/roles/sf-opensearch/tasks/create_users.yml deleted file mode 100644 index 752a12f9a..000000000 --- a/ansible/roles/sf-opensearch/tasks/create_users.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# note: user dict looks like: -# internal_users: -# name: -# role: -# password: - -- name: Generate hash for main SF host - vars: - internal_users: - admin: - role: admin - password: "{{ opensearch['opensearch_password'] | default(opensearch_password) }}" - kibanaserver: - role: kibanaserver - password: "{{ opensearch['kibanaserver_password'] | default(opensearch_kibanaserver_password) }}" - logstash: - role: logstash - password: "{{ opensearch['logstash_password'] | default(opensearch_logstash_password) }}" - zuul: - role: zuul - password: "{{ opensearch['zuul_password'] | default(opensearch_zuul_password) }}" - # readonly user - kibana: - role: kibana - password: "{{ opensearch['readonly_password'] | default(opensearch_readonly_password) }}" - with_dict: "{{ internal_users }}" - include_tasks: gen_user_pass.yml - no_log: "{{ show_hidden_logs }}" - -- name: Set opensearch readonly user name - set_fact: - opensearch_readonly_user: "{{ opensearch['readonly_user'] | default(opensearch_readonly_user) }}" - opensearch_readonly_password: "{{ opensearch['readonly_password'] | default(opensearch_readonly_password) }}" diff --git a/ansible/roles/sf-opensearch/tasks/disable.yml b/ansible/roles/sf-opensearch/tasks/disable.yml deleted file mode 100644 index 4900e52bf..000000000 --- a/ansible/roles/sf-opensearch/tasks/disable.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Disable services - service: - name: opensearch - state: stopped - enabled: false - ignore_errors: true - -- name: Delete containers - include_role: - name: sf-container - tasks_from: delete.yaml - loop: "{{ opensearch_components }}" - when: erase == True diff --git a/ansible/roles/sf-opensearch/tasks/gen_user_pass.yml b/ansible/roles/sf-opensearch/tasks/gen_user_pass.yml deleted file mode 100644 index 3ce644764..000000000 --- a/ansible/roles/sf-opensearch/tasks/gen_user_pass.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Set OpenSearch users as a fact - internal user - set_fact: - "opensearch_{{ item.key }}": "{{ item.value.password }}" - "opensearch_{{ item.key }}_hash": "{{ item.value.password | password_hash('bcrypt', rounds=12) }}" - no_log: "{{ show_hidden_logs }}" diff --git a/ansible/roles/sf-opensearch/tasks/get_logs.yml b/ansible/roles/sf-opensearch/tasks/get_logs.yml deleted file mode 100644 index 9ce2eaf0f..000000000 --- a/ansible/roles/sf-opensearch/tasks/get_logs.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Dump service logs - shell: journalctl -u opensearch > {{ elk_log_dir }}/opensearch.log - ignore_errors: true - -- name: Fetch service logs - synchronize: - mode: pull - src: "{{ elk_log_dir }}/" - dest: "{{ log_dest }}/log" - ignore_errors: true diff --git a/ansible/roles/sf-opensearch/tasks/install.yml b/ansible/roles/sf-opensearch/tasks/install.yml deleted file mode 100644 index 781c7c8b6..000000000 --- a/ansible/roles/sf-opensearch/tasks/install.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- name: Change Opensearch certificates new name - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - remote_src: true - loop: - - src: "{{ elk_stack_certs }}/elasticsearch-admin.crt" - dest: "{{ elk_stack_certs }}/opensearch-admin.crt" - - src: "{{ elk_stack_certs }}/elasticsearch-admin.key" - dest: "{{ elk_stack_certs }}/opensearch-admin.key" - ignore_errors: true - -- name: Remove old certs - file: - patch: "{{ item }}" - remote_src: true - loop: - - "{{ elk_stack_certs }}/elasticsearch-admin.crt" - - "{{ elk_stack_certs }}/elasticsearch-admin.key" - ignore_errors: true - -- include_tasks: "{{ sf_tasks_dir }}/create_user.yml" - vars: - user_name: opensearch - -- name: Get opensearch uid - command: id -u opensearch - register: _opensearch_uid - -- name: Expose opensearch uid - set_fact: - opensearch_uid: "{{ _opensearch_uid.stdout }}" - -- name: Pull image - include_role: - name: sf-container - tasks_from: install.yaml - loop: "{{ opensearch_components }}" diff --git a/ansible/roles/sf-opensearch/tasks/main.yml b/ansible/roles/sf-opensearch/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-opensearch/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-opensearch/tasks/postconf.yml b/ansible/roles/sf-opensearch/tasks/postconf.yml deleted file mode 100644 index efbd8b87a..000000000 --- a/ansible/roles/sf-opensearch/tasks/postconf.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -- name: Wait for Opensearch service - wait_for: - host: '{{ opensearch_host }}' - port: '{{ opensearch_http_port }}' - delay: 10 - timeout: 600 - -# Configure ISM - Index State Management -# NOTE: the '*' will be added to the index name on creating the policy. -- name: Configure Index State Management - include_tasks: create_ism.yml - loop: - - policy_name: "logstash-policy" - index_name: "logstash-" - retention_time: "{{ logstash_retention_days | default(14) }}d" - - policy_name: "zuul-policy" - index_name: "zuul." - retention_time: "{{ zuul_retention_days | default(180) }}d" - # Create universal ISM policy that does not provide index - - policy_name: "universal-policy" - retention_time: "14d" - -- name: Configure Opensearch replicas - uri: - url: '{{ opensearch_internal_url }}/_template/default_settings' - method: PUT - body_format: json - client_cert: "{{ elk_stack_certs }}/opensearch-admin.crt" - client_key: "{{ elk_stack_certs }}/opensearch-admin.key" - body: '{ - "template" : "*", - "settings" : { - "number_of_replicas" : {{ opensearch_replicas }} - } - }' - status_code: 200 - validate_certs: true - -- name: Check if indexes replicas number should be updated - uri: - url: '{{ opensearch_internal_url }}/_settings?pretty' - method: GET - status_code: 200 - client_cert: "{{ elk_stack_certs }}/opensearch-admin.crt" - client_key: "{{ elk_stack_certs }}/opensearch-admin.key" - validate_certs: true - register: json_response - -- name: Update number of replicas for all existing indexes - uri: - url: "{{ opensearch_internal_url }}/*/_settings" - method: PUT - body_format: json - client_cert: "{{ elk_stack_certs }}/opensearch-admin.crt" - client_key: "{{ elk_stack_certs }}/opensearch-admin.key" - body: '{ "index": - { "number_of_replicas": {{ opensearch_replicas }} - } - }' - status_code: 200 - validate_certs: true - when: json_response.json != {} diff --git a/ansible/roles/sf-opensearch/tasks/setup.yml b/ansible/roles/sf-opensearch/tasks/setup.yml deleted file mode 100644 index 6769ad119..000000000 --- a/ansible/roles/sf-opensearch/tasks/setup.yml +++ /dev/null @@ -1,181 +0,0 @@ ---- -- name: Check container image version - include_role: - name: sf-container - tasks_from: check_version.yaml - loop: "{{ opensearch_components }}" - -- name: Remove old node directory when moving to Opensearch - file: - path: "{{ elk_data_dir }}" - state: absent - when: - - sf_previous_version < 3.7 - -- name: Create cert dir - file: - path: "{{ item }}" - state: directory - recurse: true - owner: "{{ owner }}" - group: "{{ group }}" - loop: - - "{{ elk_stack_certs }}" - - "{{ elk_data_dir }}" - - "{{ elk_log_dir }}" - - "{{ elk_log_dir }}/others" - -- name: Rename Opensearch admin key - copy: - src: "{{ elastic_admin_key }}" - dest: "{{ elastic_admin_key }}-default" - vars: - elastic_admin_key: /var/lib/software-factory/bootstrap-data/certs/opensearch-admin.key - delegate_to: "{{ install_server_host }}" - -- name: Convert opensearch admin private key to pkcs8 - shell: | - openssl pkcs8 -in {{ sf_cert_path }}/opensearch-admin.key-default -topk8 -out {{sf_cert_path }}/opensearch-admin.key -nocrypt - vars: - sf_cert_path: /var/lib/software-factory/bootstrap-data/certs - delegate_to: "{{ install_server_host }}" - -- name: Copy certs - copy: - src: "/var/lib/software-factory/bootstrap-data/certs/{{ item }}" - dest: "{{ elk_stack_certs }}/{{ item }}" - mode: "0644" - owner: "{{ owner }}" - group: "{{ group }}" - loop: - - opensearch-admin.crt - - opensearch-admin.key - - localCA.pem - -# This is required to set opendistro_security.authcz.elasticsearch_dn -- name: Take opensearch subject - shell: | - openssl x509 -subject -nameopt RFC2253 -noout -in {{ elk_stack_certs }}/opensearch-admin.crt | sed 's/subject= //g' | sed 's/subject=//g' - register: elk_cert_subject - -# https://opensearch.org/docs/opensearch/install/important-settings/ -- name: Set important settings for Opensearch container - sysctl: - name: vm.max_map_count - value: "262144" - state: present - reload: true - -# Setup ELK stack users -- include_tasks: create_users.yml - -- name: Create reconfiguration script - copy: - content: >- - #!/bin/bash - - podman exec opensearch - bash -c 'JAVA_HOME={{ opensearch_container_dir }}/jdk - {{ opensearch_container_dir }}/plugins/opensearch-security/tools/securityadmin.sh - -cd {{ opensearch_container_securityconfig_dir }}/ - -icl -nhnv -cacert {{ opensearch_container_config_dir }}/certificates/localCA.pem - -cert {{ opensearch_container_config_dir }}/certificates/opensearch-admin.crt - -key {{ opensearch_container_config_dir }}/certificates/opensearch-admin.key - -h {{ ansible_default_ipv4.address }}' - dest: /usr/local/bin/reconfigure-opensearch.sh - mode: '0755' - -- name: Set the security plugin configuration - template: - src: "{{ item }}.j2" - dest: "{{ elk_config_dir }}/{{ item }}" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0640" - loop: - - internal_users.yml - - tenants.yml - - config.yml - - roles.yml - - roles_mapping.yml - - action_groups.yml - - nodes_dn.yml - - whitelist.yml - notify: - - ensure started opensearch - - wait for opensearch service - - reconfigure security plugin in containers - -- name: Ensure old curator cron absent - cron: - name: "Delete old logstash indexes" - state: absent - -- name: Create log4j2.properties file - copy: - src: log4j2.properties - dest: "/etc/opensearch/log4j2.properties" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0640" - notify: - - ensure started opensearch - - wait for opensearch service - -- name: Configure Opensearch - template: - src: opensearch-config.yml.j2 - dest: "{{ elk_config_dir }}/opensearch.yml" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0644" - notify: - - restart opensearch - - ensure started opensearch - - wait for opensearch service - - reconfigure security plugin in containers - -- name: Get opensearch uid - command: id -u opensearch - register: _opensearch_uid - -- name: Expose opensearch uid - set_fact: - opensearch_uid: "{{ _opensearch_uid.stdout }}" - -- name: Create container - Opensearch - include_role: - name: sf-container - tasks_from: setup.yaml - loop: "{{ opensearch_components }}" - -- name: Ensure we restart the service when the fqdn is updated - service: - name: opensearch - state: stopped - when: update_fqdn - -- name: Ensure the service is started - service: - name: opensearch - state: started - enabled: "yes" - -# NOTE: Below tasks comes from handlers, but they needs to be executed now, -# because Opensearch and Opensearch Dashboards services can be deployed in -# wrong order, so the handlers from Dashboards role will wait for Opensearch -# to be up and running, where Opensearch did not synchronize the security -# plugin yet. -- name: wait for opensearch service - uri: - url: "{{ opensearch_internal_url }}" - method: GET - validate_certs: false - status_code: "401" - register: _opensearch_service_status - until: "(_opensearch_service_status is successful) and (_opensearch_service_status.status == 401)" - retries: 30 - delay: 10 - -- name: reconfigure security plugin in containers - command: /usr/local/bin/reconfigure-opensearch.sh diff --git a/ansible/roles/sf-opensearch/tasks/upgrade.yml b/ansible/roles/sf-opensearch/tasks/upgrade.yml deleted file mode 100644 index 38012932a..000000000 --- a/ansible/roles/sf-opensearch/tasks/upgrade.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Get available service facts - service_facts: - -- name: Stop Opensearch service - systemd: - name: opensearch - state: stopped - when: "'opensearch.service' in services" - -- name: Remove old node directory when moving to Opensearch - file: - path: "{{ elk_data_dir }}" - state: absent - when: - - sf_previous_version < 3.7 - -- name: Upgrade container - include_role: - name: sf-container - tasks_from: upgrade.yaml - loop: "{{ opensearch_components }}" diff --git a/ansible/roles/sf-opensearch/templates/action_groups.yml.j2 b/ansible/roles/sf-opensearch/templates/action_groups.yml.j2 deleted file mode 100644 index 7c40612b8..000000000 --- a/ansible/roles/sf-opensearch/templates/action_groups.yml.j2 +++ /dev/null @@ -1,3 +0,0 @@ -_meta: - type: "actiongroups" - config_version: 2 diff --git a/ansible/roles/sf-opensearch/templates/config.yml.j2 b/ansible/roles/sf-opensearch/templates/config.yml.j2 deleted file mode 100644 index 952c14b53..000000000 --- a/ansible/roles/sf-opensearch/templates/config.yml.j2 +++ /dev/null @@ -1,43 +0,0 @@ ---- -_meta: - config_version: 2 - type: config -config: - dynamic: - http: - anonymous_auth_enabled: false - xff: - enabled: false -{% if readonly_user_autologin is defined and readonly_user_autologin | lower == 'basic' %} - remoteIpHeader: "x-forwarded-for" - internalProxies: "{{ fqdn }}" -{% endif %} - authc: - basic_internal_auth_domain: - http_enabled: true - transport_enabled: true - order: 0 - authentication_backend: - type: intern - http_authenticator: - type: basic - challenge: false - - openid_auth_domain: - http_enabled: true - transport_enabled: true - order: 1 - http_authenticator: - type: openid - challenge: false - config: - subject_key: preferred_username - roles_key: roles - openid_connect_url: https://{{ fqdn }}/auth/realms/{{ keycloak_default_realm }}/.well-known/openid-configuration - openid_connect_idp.enable_ssl: true - openid_connect_idp.pemtrustedcas_filepath: "{{ opensearch_container_certs_dir }}/localCA.pem" - openid_connect_idp.verify_hostname: false - skip_users: - - kibanaserver - authentication_backend: - type: noop diff --git a/ansible/roles/sf-opensearch/templates/internal_users.yml.j2 b/ansible/roles/sf-opensearch/templates/internal_users.yml.j2 deleted file mode 100644 index 284ac21ab..000000000 --- a/ansible/roles/sf-opensearch/templates/internal_users.yml.j2 +++ /dev/null @@ -1,40 +0,0 @@ ---- -_meta: - type: "internalusers" - config_version: 2 -admin: - hash: {{ opensearch_admin_hash }} - reserved: true - backend_roles: - - admin - description: Admin user - hidden: true -kibanaserver: - hash: {{ opensearch_kibanaserver_hash }} - reserved: true - backend_roles: - - kibanauser - description: Kibanaserver user - hidden: true -logstash: - hash: {{ opensearch_logstash_hash }} - reserved: false - backend_roles: - - logstash - description: Logstash user - hidden: true -zuul: - hash: {{ opensearch_zuul_hash }} - reserved: false - backend_roles: - - admin - description: Zuul user - hidden: true -# exception in Elasticsearch (https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles/#predefined-roles) -# FIXME: switch to the official roles when bug is fixed. -{{ opensearch_readonly_user | default('kibana') }}: - hash: {{ opensearch_kibana_hash }} - reserved: true - backend_roles: - - sf_opensearch_dashboards_user - description: Kibana viewer user diff --git a/ansible/roles/sf-opensearch/templates/ism.json.j2 b/ansible/roles/sf-opensearch/templates/ism.json.j2 deleted file mode 100644 index 39285e964..000000000 --- a/ansible/roles/sf-opensearch/templates/ism.json.j2 +++ /dev/null @@ -1,71 +0,0 @@ -{% if 'index_name' in item %} -{ - "policy": { - "policy_id": "{{ item.policy_name }}", - "description": "Delete index after {{ item.retention_time | default('14d') }} for {{ item.index_name }}", - "default_state": "hot", - "states": [ - { - "name": "hot", - "actions": [], - "transitions": [ - { - "state_name": "delete", - "conditions": { - "min_index_age": "{{ item.retention_time | default('14d') }}" - } - } - ] - }, - { - "name": "delete", - "actions": [ - { - "delete": {} - } - ], - "transitions": [] - } - ], - "ism_template": [ - { - "index_patterns": [ - "{{ item.index_name }}*" - ], - "priority": 1 - } - ] - } -} -{% else %} -{ - "policy": { - "policy_id": "{{ item.policy_name }}", - "description": "Delete all data after {{ item.retention_time | default('14d') }}", - "default_state": "hot", - "states": [ - { - "name": "hot", - "actions": [], - "transitions": [ - { - "state_name": "delete", - "conditions": { - "min_index_age": "{{ item.retention_time | default('14d') }}" - } - } - ] - }, - { - "name": "delete", - "actions": [ - { - "delete": {} - } - ], - "transitions": [] - } - ] - } -} -{% endif %} diff --git a/ansible/roles/sf-opensearch/templates/nodes_dn.yml.j2 b/ansible/roles/sf-opensearch/templates/nodes_dn.yml.j2 deleted file mode 100644 index 7f8304cf0..000000000 --- a/ansible/roles/sf-opensearch/templates/nodes_dn.yml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -_meta: - type: "nodesdn" - config_version: 2 - -# Define nodesdn mapping name and corresponding values -# cluster1: -# nodes_dn: -# - CN=*.example.com diff --git a/ansible/roles/sf-opensearch/templates/opensearch-config.yml.j2 b/ansible/roles/sf-opensearch/templates/opensearch-config.yml.j2 deleted file mode 100644 index 546ecd9f6..000000000 --- a/ansible/roles/sf-opensearch/templates/opensearch-config.yml.j2 +++ /dev/null @@ -1,33 +0,0 @@ ---- -cluster.name: {{ fqdn | replace('.', '_') }} -node.name: {{ inventory_hostname }} -path.data: {{ opensearch_container_dir }}/data -path.logs: {{ elk_log_dir }} -network.host: 0.0.0.0 -http.port: 9200 - -node.roles: master,data,ingest -discovery.type: single-node -cluster.routing.allocation.disk.threshold_enabled: false - -# along with the memlock settings below, disables swapping -bootstrap.memory_lock: true - -plugins.security.ssl.transport.enforce_hostname_verification: false -plugins.security.ssl.http.enabled: true -plugins.security.allow_default_init_securityindex: true -plugins.security.allow_unsafe_democertificates: false - -plugins.security.ssl.transport.pemcert_filepath: {{ opensearch_container_certs_dir }}/opensearch-admin.crt -plugins.security.ssl.transport.pemkey_filepath: {{ opensearch_container_certs_dir }}/opensearch-admin.key -plugins.security.ssl.transport.pemtrustedcas_filepath: {{ opensearch_container_certs_dir }}/localCA.pem -plugins.security.ssl.http.pemcert_filepath: {{ opensearch_container_certs_dir }}/opensearch-admin.crt -plugins.security.ssl.http.pemkey_filepath: {{ opensearch_container_certs_dir }}/opensearch-admin.key -plugins.security.ssl.http.pemtrustedcas_filepath: {{ opensearch_container_certs_dir }}/localCA.pem -plugins.security.authcz.admin_dn: - - {{ elk_cert_subject.stdout | default('CN=opensearch.sftests.com,O=OpensearchSoftwareFactory,L=WROCLAW,ST=DOLNOSLASKIE,C=PL') }} - -plugins.security.enable_snapshot_restore_privilege: true -plugins.security.check_snapshot_restore_write_privileges: true -plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] -plugins.security.system_indices.enabled: true diff --git a/ansible/roles/sf-opensearch/templates/roles.yml.j2 b/ansible/roles/sf-opensearch/templates/roles.yml.j2 deleted file mode 100644 index 4e29bd947..000000000 --- a/ansible/roles/sf-opensearch/templates/roles.yml.j2 +++ /dev/null @@ -1,32 +0,0 @@ -_meta: - type: "roles" - config_version: 2 -# Opensearch comes with pre-defined roles. -# https://opensearch.org/docs/latest/security-plugin/access-control/users-roles/#predefined-roles - -# Opensearch comes with pre-defined actions groups (that are logical groups of permission) -# https://opensearch.org/docs/latest/security-plugin/access-control/default-action-groups/ - -# Opensearch permissions are -# https://opensearch.org/docs/latest/security-plugin/access-control/permissions/ - -# This roles is permissif but is only applied to users connected via Keycloak. -# KC add that role into the JWT 'roles' claim list. - -sf_opensearch_dashboards_user: - reserved: true - cluster_permissions: - - "cluster_composite_ops_ro" - index_permissions: - - index_patterns: - - "*" - allowed_actions: - - "read" - - "get" - - "search" - - "write" - tenant_permissions: - - tenant_patterns: - - "*" - allowed_actions: - - "kibana_all_write" diff --git a/ansible/roles/sf-opensearch/templates/roles_mapping.yml.j2 b/ansible/roles/sf-opensearch/templates/roles_mapping.yml.j2 deleted file mode 100644 index bed7b9307..000000000 --- a/ansible/roles/sf-opensearch/templates/roles_mapping.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ ---- -_meta: - type: "rolesmapping" - config_version: 2 - -all_access: - reserved: false - backend_roles: - - "admin" - description: "Maps admin to all_access" - -own_index: - reserved: false - users: - - "*" - description: "Allow full access to an index named like the username" - -logstash: - reserved: false - backend_roles: - - "logstash" - -kibana_user: - reserved: false - backend_roles: - - "kibanauser" - description: "Maps kibanauser to kibana_user" - -readall: - reserved: false - backend_roles: - - "readall" - -manage_snapshots: - reserved: false - backend_roles: - - "snapshotrestore" - -kibana_server: - reserved: true - users: - - "kibanaserver" - -sf_opensearch_dashboards_user: - reserved: false - backend_roles: - - "sf_opensearch_dashboards_user" diff --git a/ansible/roles/sf-opensearch/templates/tenants.yml.j2 b/ansible/roles/sf-opensearch/templates/tenants.yml.j2 deleted file mode 100644 index 93b510dd1..000000000 --- a/ansible/roles/sf-opensearch/templates/tenants.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ ---- -_meta: - type: "tenants" - config_version: 2 diff --git a/ansible/roles/sf-opensearch/templates/whitelist.yml.j2 b/ansible/roles/sf-opensearch/templates/whitelist.yml.j2 deleted file mode 100644 index 1d54ff841..000000000 --- a/ansible/roles/sf-opensearch/templates/whitelist.yml.j2 +++ /dev/null @@ -1,69 +0,0 @@ ---- -_meta: - type: "whitelist" - config_version: 2 - -# Description: -# enabled - feature flag. -# if enabled is false, the whitelisting feature is removed. -# This is like removing the check that checks if an API is whitelisted. -# This is equivalent to continuing with the usual access control checks, and removing all the code that implements whitelisting. -# if enabled is true, then all users except SuperAdmin can access only the APIs in requests -# SuperAdmin can access all APIs. -# SuperAdmin is defined by the SuperAdmin certificate, which is configured in the opensearch.yml setting: plugins.security.authcz.admin_dn: -# Refer to the example setting in opensearch.yml.example, and the opendistro documentation to know more about configuring SuperAdmin. -# -# requests - map of whitelisted endpoints, and the whitelisted HTTP requests for those endpoints - -# Examples showing how to configure this yml file (make sure the _meta data from above is also there): -# Example 1: -# To enable whitelisting and whitelist GET /_cluster/settings -# -#config: -# enabled: true -# requests: -# /_cluster/settings: -# - GET -# -# Example 2: -# If you want to whitelist multiple request methods for /_cluster/settings (GET,PUT): -# -#config: -# enabled: true -# requests: -# /_cluster/settings: -# - GET -# - PUT -# -# Example 3: -# If you want to whitelist other APIs as well, for example GET /_cat/nodes, and GET /_cat/shards: -# -#config: -# enabled: true -# requests: -# /_cluster/settings: -# - GET -# - PUT -# /_cat/nodes: -# - GET -# /_cat/shards: -# - GET -# -# Example 4: -# If you want to disable the whitelisting feature, set enabled to false. -# enabled: false -# requests: -# /_cluster/settings: -# - GET -# -#At this point, all APIs become whitelisted because the feature to whitelist is off, so requests is irrelevant. - - -#this name must be config -config: - enabled: false - requests: - /_cluster/settings: - - GET - /_cat/nodes: - - GET diff --git a/ansible/roles/sf-telegraf/defaults/main.yml b/ansible/roles/sf-telegraf/defaults/main.yml deleted file mode 100644 index 7003f304c..000000000 --- a/ansible/roles/sf-telegraf/defaults/main.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -role_actions: - - disable - - install - - setup - - get_logs - -role_package: telegraf - -container_uid: 1000 -container_gid: 1000 -owner: "{{ role_package }}" -group: "{{ role_package }}" - -influxdb_host: influxdb.example.com -influxdb_port: 8086 -telegraf_port: 8125 -telegraf_influxdb_user: telegraf -telegraf_influxdb_password: CHANGE_ME - -telegraf_version: 1.24.4 -telegraf_container_version: "{{ telegraf_version }}-2" - -telegraf_config_dir: /etc/telegraf - -telegraf_components: - - service: "{{ role_package }}" - image: "quay.io/software-factory/telegraf:{{ telegraf_container_version }}" - params: >- - --network host - --uidmap 0:4000:999 - --uidmap {{ container_uid }}:{{ telegraf_uid | default(1000) }}:1 - --volume {{ telegraf_config_dir }}:{{ telegraf_config_dir }}:Z - --volume /etc/ssl/certs/ca-bundle.crt:/etc/ssl/certs/ca-bundle.crt diff --git a/ansible/roles/sf-telegraf/handlers/main.yml b/ansible/roles/sf-telegraf/handlers/main.yml deleted file mode 100644 index 3188cb7ee..000000000 --- a/ansible/roles/sf-telegraf/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Restart telegraf - service: - name: telegraf - state: restarted diff --git a/ansible/roles/sf-telegraf/meta/main.yml b/ansible/roles/sf-telegraf/meta/main.yml deleted file mode 100644 index c648e7699..000000000 --- a/ansible/roles/sf-telegraf/meta/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -galaxy_info: - description: Telegraf service - company: Red Hat - license: Apache - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 - galaxy_tags: - - software-factory diff --git a/ansible/roles/sf-telegraf/tasks/disable.yml b/ansible/roles/sf-telegraf/tasks/disable.yml deleted file mode 100644 index 235ec4e0b..000000000 --- a/ansible/roles/sf-telegraf/tasks/disable.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: Disable services - service: - name: '{{ item }}' - state: stopped - enabled: "no" - loop: - - '{{ role_package }}' - ignore_errors: "yes" - -- block: - - name: "Check if {{ role_package }} container exists" - command: "podman container exists {{ role_package }}" - ignore_errors: yes - register: _container_exists - changed_when: no - failed_when: _container_exists.rc not in [0, 1] - become: true - - - name: Delete containers - include_role: - name: sf-container - tasks_from: delete.yaml - loop: "{{ telegraf_components }}" - when: _container_exists.rc == 0 - - - name: Remove files - file: - path: "{{ item }}" - state: absent - loop: - - '/etc/{{ role_package }}' - - - name: Remove packages - yum: - name: '{{ item }}' - state: absent - loop: - - '{{ role_package }}' - when: _container_exists.rc != 0 - - when: erase == True diff --git a/ansible/roles/sf-telegraf/tasks/get_logs.yml b/ansible/roles/sf-telegraf/tasks/get_logs.yml deleted file mode 100644 index f66b897fd..000000000 --- a/ansible/roles/sf-telegraf/tasks/get_logs.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Getting container logs - include_role: - name: sf-container - tasks_from: get_logs.yaml - loop: - - container: "{{ role_package }}" - options: "" - target: "{{ log_dest }}" diff --git a/ansible/roles/sf-telegraf/tasks/install.yml b/ansible/roles/sf-telegraf/tasks/install.yml deleted file mode 100644 index 965497ad3..000000000 --- a/ansible/roles/sf-telegraf/tasks/install.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Remove telegraf packages - yum: - name: "telegraf" - state: absent - -- name: Pull image - include_role: - name: sf-container - tasks_from: install.yaml - loop: "{{ telegraf_components }}" diff --git a/ansible/roles/sf-telegraf/tasks/main.yml b/ansible/roles/sf-telegraf/tasks/main.yml deleted file mode 100644 index bbdfe1414..000000000 --- a/ansible/roles/sf-telegraf/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- include_tasks: "{{ role_action }}.yml" - when: role_action in role_actions diff --git a/ansible/roles/sf-telegraf/tasks/setup.yml b/ansible/roles/sf-telegraf/tasks/setup.yml deleted file mode 100644 index 0ed0210ca..000000000 --- a/ansible/roles/sf-telegraf/tasks/setup.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- -- include_tasks: "{{ sf_tasks_dir }}/create_user.yml" - vars: - user_name: "{{ owner }}" - -- name: Get {{ owner }} uid - command: id -u {{ owner }} - register: _telegraf_id - -- name: Get {{ owner }} gid - command: id -g {{ owner }} - register: _telegraf_gid - -- name: Setting {{ owner }} uid var - set_fact: - telegraf_uid: "{{ _telegraf_id.stdout }}" - telegraf_gid: "{{ _telegraf_gid.stdout }}" - -- name: Check container image version - include_role: - name: sf-container - tasks_from: check_version.yaml - loop: "{{ telegraf_components }}" - -- name: Create container directory - include_role: - name: sf-container - tasks_from: create_directory.yaml - loop: - - path: "{{ telegraf_config_dir }}" - recurse: "yes" - owner: "{{ owner }}" - group: "{{ group }}" - mode: "0750" - -- name: Apply selinux port labelling - seport: - ports: "{{ telegraf_port }}" - proto: "udp" - setype: "http_port_t" - state: present - when: - - ansible_virtualization_type not in ["podman", "systemd-nspawn"] - - ansible_selinux.status == "enabled" - -- name: Setup config files - template: - src: "telegraf.conf.j2" - dest: "{{ telegraf_config_dir }}/{{ item }}" - loop: - - telegraf.conf - -- name: Create container - include_role: - name: sf-container - tasks_from: setup.yaml - loop: "{{ telegraf_components }}" - -- name: Restart telegraf when fqdn is updated - systemd: - name: "{{ role_package }}" - state: restarted - when: update_fqdn - -- name: Start service - systemd: - name: "{{ role_package }}" - state: started - daemon_reload: true - enabled: true diff --git a/ansible/roles/sf-telegraf/tasks/upgrade.yml b/ansible/roles/sf-telegraf/tasks/upgrade.yml deleted file mode 100644 index f2c2f04b9..000000000 --- a/ansible/roles/sf-telegraf/tasks/upgrade.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Stop the service - service: - name: "{{ role_package }}" - state: stopped - -- name: Remove {{ role_package }} rpm package - yum: - name: "{{ role_package }}" - state: absent - -- name: Upgrade container - include_role: - name: sf-container - tasks_from: upgrade.yaml - loop: "{{ telegraf_components }}" diff --git a/ansible/roles/sf-telegraf/templates/telegraf.conf.j2 b/ansible/roles/sf-telegraf/templates/telegraf.conf.j2 deleted file mode 100644 index d8cc29d45..000000000 --- a/ansible/roles/sf-telegraf/templates/telegraf.conf.j2 +++ /dev/null @@ -1,38 +0,0 @@ -[global_tags] -[agent] - interval = "10s" - round_interval = true - metric_batch_size = 1000 - metric_buffer_limit = 10000 - collection_jitter = "0s" - flush_interval = "10s" - flush_jitter = "0s" - precision = "" - debug = false - quiet = false - logfile = "" - hostname = "{{ ansible_fqdn }}" - omit_hostname = false - -[[outputs.influxdb]] - urls = ["https://{{ influxdb_host }}:{{ influxdb_port }}"] - database = "telegraf" - retention_policy = "" - write_consistency = "any" - timeout = "5s" - username = "{{ telegraf_influxdb_user }}" - password = "{{ telegraf_influxdb_password }}" - skip_database_creation = true - -# Statsd Server -[[inputs.statsd]] - service_address = ":8125" - delete_gauges = true - delete_counters = true - delete_sets = true - delete_timings = true - percentiles = [90.0] - metric_separator = "." - parse_data_dog_tags = false - allowed_pending_messages = 10000 - percentile_limit = 1000 diff --git a/refarch/allinone.yaml b/refarch/allinone.yaml index c400932a5..8366030ed 100644 --- a/refarch/allinone.yaml +++ b/refarch/allinone.yaml @@ -17,16 +17,9 @@ inventory: - zuul-executor - zuul-fingergw - zuul-web - - log-classify - gerritbot - nodepool-launcher - nodepool-builder - murmur - - opensearch - - log-processing - - opensearch-dashboards - - firehose - - influxdb - - grafana - hound - cgit diff --git a/scripts/submit_and_wait.py b/scripts/submit_and_wait.py index 781a90cb8..01fd77d83 100755 --- a/scripts/submit_and_wait.py +++ b/scripts/submit_and_wait.py @@ -89,7 +89,8 @@ def main(): gr_opts = '-v' if args.review_id: - print(execute("/usr/local/bin/git-review %s -d %s" % (gr_opts, args.review_id))) + print(execute("/usr/local/bin/git-review %s -d %s" + % (gr_opts, args.review_id))) sha = execute("git log -n1 --pretty=format:%H") else: # Submit change diff --git a/sfconfig/inventory.py b/sfconfig/inventory.py index 90355c9fb..479a3c806 100644 --- a/sfconfig/inventory.py +++ b/sfconfig/inventory.py @@ -505,7 +505,8 @@ def enable_action(args): testinfra_tests = sfconfig.utils.list_testinfra() for host in args.glue["inventory"]: logfile = "/var/log/software-factory/testinfra.log" - testinfra = ["/usr/local/bin/py.test", "-v", "--log-file=%s" % logfile] + testinfra = ["/usr/local/bin/py.test", "-v", "--log-file=%s" + % logfile] tests = [] if "install-server" not in host["roles"]: testinfra.append("--connection=ssh") diff --git a/testinfra/test_elasticsearch.py b/testinfra/test_elasticsearch.py deleted file mode 100644 index a728f83bf..000000000 --- a/testinfra/test_elasticsearch.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class Testopensearch: - def test_service_running_and_enabled(self, host): - opensearch = host.service("opensearch") - assert opensearch.is_running - assert opensearch.is_enabled diff --git a/testinfra/test_etherpad.py b/testinfra/test_etherpad.py index 1f6b6e817..697fba80d 100644 --- a/testinfra/test_etherpad.py +++ b/testinfra/test_etherpad.py @@ -16,5 +16,5 @@ def test_service_running_and_enabled(self, host): etherpad = host.service("etherpad") assert etherpad.is_running assert etherpad.is_enabled - skt = host.socket("tcp://127.0.0.1:9001") + skt = host.socket("tcp://0.0.0.0:9001") assert skt.is_listening diff --git a/testinfra/test_grafana.py b/testinfra/test_grafana.py deleted file mode 100644 index b16b435ab..000000000 --- a/testinfra/test_grafana.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class TestGrafana: - def test_service_running_and_enabled(self, host): - service = host.service("grafana-server") - assert service.is_running - assert service.is_enabled - - def test_service_endpoint(self, host): - assert host.socket("tcp://3000").is_listening diff --git a/testinfra/test_influxdb.py b/testinfra/test_influxdb.py deleted file mode 100644 index 94852c615..000000000 --- a/testinfra/test_influxdb.py +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class TestInfluxdb: - def test_influxdb_service_running_and_enabled(self, host): - service = host.service('influxdb') - assert service.is_running - assert service.is_enabled - - def test_service_listening(self, host): - assert host.socket("tcp://8086").is_listening - assert host.socket("tcp://127.0.0.1:8088").is_listening diff --git a/testinfra/test_kibana.py b/testinfra/test_kibana.py deleted file mode 100644 index d2950c0a3..000000000 --- a/testinfra/test_kibana.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class TestOpensearchDashboards: - def test_service_running_and_enabled(self, host): - service = host.service("opensearch-dashboards") - assert service.is_running - assert service.is_enabled diff --git a/testinfra/test_murmur.py b/testinfra/test_murmur.py index a881e9710..905d9b9a8 100644 --- a/testinfra/test_murmur.py +++ b/testinfra/test_murmur.py @@ -13,6 +13,6 @@ class TestMurmur: def test_service_running_and_enabled(self, host): - service = host.service("umurmurd") + service = host.service("umurmur") assert service.is_running assert service.is_enabled diff --git a/testinfra/test_telegraf-statsd.py b/testinfra/test_telegraf-statsd.py deleted file mode 100644 index 3088d95d9..000000000 --- a/testinfra/test_telegraf-statsd.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import utils - - -class TestTelegrafStatsd(utils.Base): - def test_telegraft_statsd_intput_plugin(self, host): - enabled_roles = self.enabled_roles() - if 'nodepool' in enabled_roles or 'zuul' in enabled_roles: - statsd_config = '/etc/telegraf/telegraf.d/statsd.conf' - assert host.file(statsd_config).exists - assert host.socket("udp://8125").is_listening diff --git a/testinfra/test_telegraf.py b/testinfra/test_telegraf.py deleted file mode 100644 index 510d2ae40..000000000 --- a/testinfra/test_telegraf.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class TestTelegraf: - def test_telegraf_service_running_and_enabled(self, host): - service = host.service('telegraf') - assert service.is_running - assert service.is_enabled