From fa274513f577beb97a55c555806521ecf187f21a Mon Sep 17 00:00:00 2001 From: sbbroot <86356638+sbbroot@users.noreply.github.com> Date: Thu, 19 May 2022 11:36:02 +0200 Subject: [PATCH] Optimize Grafana dashboards downloading (#3131) (#3150) --- .../src/config/config.py | 20 ++- .../src/mode/base_mode.py | 11 +- .../tests/config/test_config.py | 16 ++- .../tests/config/test_manifest_reader.py | 15 ++- .../tests/data/config.py | 81 +++++++++++- .../tests/data/manifest_reader.py | 122 ++++++++++++------ cli/src/ansible/AnsibleInventoryCreator.py | 4 +- cli/src/commands/Apply.py | 4 +- 8 files changed, 211 insertions(+), 62 deletions(-) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py index b6dde5b00d..b1bd8228ee 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py @@ -37,11 +37,11 @@ def __init__(self, argv: List[str]): self.__add_args(argv) + self.__LINE_SIZE: int = 50 # used in printing + if not self.rerun: self.__log_info_summary() - self.__LINE_SIZE: int = 50 # used in printing - def __log_info_summary(self): """ Helper function for printing all parsed arguments @@ -61,7 +61,7 @@ def __log_info_summary(self): lines.append(f'Repos backup file: {str(self.repos_backup_file)}') if self.dest_manifest: - lines.append(f'Manifest used: {str(self.dest_manifest)}') + lines.append(f'Manifest used: {str(self.dest_manifest.absolute())}') if self.is_log_file_enabled: lines.append(f'Log file location: {str(self.log_file.absolute())}') @@ -216,7 +216,7 @@ def __add_args(self, argv: List[str]): self.rerun = args['rerun'] self.pyyaml_installed = args['pyyaml_installed'] - def __print_parsed_manifest_data(self, output: Dict[str, Any]): + def __print_parsed_manifest_data(self, requirements: Dict[str, Any], output: Dict[str, Any]): lines: List[str] = ['Manifest summary:'] lines.append('-' * self.__LINE_SIZE) @@ -231,6 +231,13 @@ def __print_parsed_manifest_data(self, output: Dict[str, Any]): for feature in output['detected-features']: lines.append(f'- {feature}') + dashboards = requirements['grafana-dashboards'] + if dashboards: + lines.append('') + lines.append('Dashboards to download:') + for dashboard in dashboards: + lines.append(f'- {dashboard}') + lines.append('-' * self.__LINE_SIZE) logging.info('\n'.join(lines)) @@ -248,5 +255,8 @@ def read_manifest(self, requirements: Dict[str, Any]): mreader = ManifestReader(self.dest_manifest) output = mreader.parse_manifest() + if 'grafana' not in output['detected-features']: + requirements['grafana-dashboards'] = [] + if self.verbose_mode: - self.__print_parsed_manifest_data(output) + self.__print_parsed_manifest_data(requirements, output) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py index cc860c1b11..5a2c2f7df6 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py @@ -229,10 +229,12 @@ def run(self): """ # add required directories self._cfg.dest_files.mkdir(exist_ok=True, parents=True) - self._cfg.dest_grafana_dashboards.mkdir(exist_ok=True, parents=True) self._cfg.dest_images.mkdir(exist_ok=True, parents=True) self._cfg.dest_packages.mkdir(exist_ok=True, parents=True) + if self._requirements['grafana-dashboards']: + self._cfg.dest_grafana_dashboards.mkdir(exist_ok=True, parents=True) + # provides tar which is required for backup logging.info('Installing base packages...') self._install_base_packages() @@ -259,9 +261,10 @@ def run(self): self.__download_files(self._requirements['files'], self._cfg.dest_files) logging.info('Done downloading files.') - logging.info('Downloading grafana dashboards...') - self.__download_grafana_dashboards() - logging.info('Done downloading grafana dashboards.') + if self._requirements['grafana-dashboards']: + logging.info('Downloading grafana dashboards...') + self.__download_grafana_dashboards() + logging.info('Done downloading grafana dashboards.') logging.info('Downloading Crane...') self.__download_crane() diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py index 081fc47b27..fda58df0dc 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py @@ -1,17 +1,21 @@ import logging from pathlib import Path +import pytest import yaml from src.config.config import Config -from tests.data.config import EXPECTED_VERBOSE_OUTPUT -from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS +from tests.data.config import EXPECTED_VERBOSE_OUTPUT, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS +from tests.data.manifest_reader import INPUT_MANIFEST_FEATURE_MAPPINGS, INPUT_MANIFEST_WITH_DASHBOARDS -def test_manifest_verbose_output(mocker, caplog): +@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC', + [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT), + (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT)]) +def test_manifest_verbose_output(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker, caplog): ''' Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` ''' - mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS)) + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC)) caplog.set_level(logging.INFO) # mock Config's init methods: @@ -23,8 +27,8 @@ def test_manifest_verbose_output(mocker, caplog): # mock required config data: config.dest_manifest = Path('/some/path') config.verbose_mode = True - config.read_manifest({}) + config.read_manifest(yaml.safe_load(DASHBOARD_REQUIREMENTS)) log_output = f'\n{"".join(caplog.messages)}\n' - assert log_output == EXPECTED_VERBOSE_OUTPUT + assert log_output == EXPECTED_OUTPUT_DOC diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py index c4f04777d2..d4c42aa283 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py @@ -1,13 +1,20 @@ from pathlib import Path +import pytest import yaml from src.config.manifest_reader import ManifestReader -from tests.data.manifest_reader import EXPECTED_FEATURE_MAPPINGS, INPUT_MANIFEST_FEATURE_MAPPINGS +from tests.data.manifest_reader import (EXPECTED_FEATURE_MAPPINGS, + EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS, + INPUT_MANIFEST_FEATURE_MAPPINGS, + INPUT_MANIFEST_WITH_DASHBOARDS) -def test_parse_manifest(mocker): +@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC', + [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS), + (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS)]) +def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, mocker): ''' Check manifest file parsing ''' - mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_MANIFEST_FEATURE_MAPPINGS)) + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC)) mreader = ManifestReader(Path('/some/path')) - assert mreader.parse_manifest() == EXPECTED_FEATURE_MAPPINGS + assert mreader.parse_manifest() == EXPECTED_OUTPUT_DOC diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py index 9275ba6aea..581c68dbe4 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py @@ -1,22 +1,97 @@ +DASHBOARD_REQUIREMENTS = """ +grafana-dashboards: + grafana_dashboard_7249: + url: 'https://grafana.com/api/dashboards/7249/revisions/1/download' + sha256: 41cc2794b1cc9fc537baf045fee12d086d23632b4c8b2e88985274bb9862e731 + grafana_dashboard_315: + url: 'https://grafana.com/api/dashboards/315/revisions/3/download' + sha256: ee46dd6e68a9950aa78e8c88ae5e565c8ebde6cbdbe08972a70f06c5486618fb + grafana_dashboard_11074: + url: 'https://grafana.com/api/dashboards/11074/revisions/9/download' + sha256: 151b23305da46eab84930e99175e1c07e375af73dbbb4b8f501ca25f5ac62785 + grafana_dashboard_405: + url: 'https://grafana.com/api/dashboards/405/revisions/8/download' + sha256: 97675027cbd5b7241e93a2b598654c4b466bc909eeb6358ba123d500094d913c + grafana_dashboard_455: + url: 'https://grafana.com/api/dashboards/455/revisions/2/download' + sha256: c66b91ab8d258b0dc005d3ee4dac3a5634a627c79cc8053875f76ab1e369a362 + grafana_dashboard_9628: + url: 'https://grafana.com/api/dashboards/9628/revisions/7/download' + sha256: c64cc38ad9ebd7af09551ee83e669a38f62a76e7c80929af5668a5852732b376 + grafana_dashboard_4279: + url: 'https://grafana.com/api/dashboards/4279/revisions/4/download' + sha256: 74d47be868da52c145240ab5586d91ace9e9218ca775af988f9d60e501907a25 + grafana_dashboard_1860: + url: 'https://grafana.com/api/dashboards/1860/revisions/23/download' + sha256: 225faab8bf35c1723af14d4c069882ccb92b455d1941c6b1cf3d95a1576c13d7 + grafana_dashboard_7589: + url: 'https://grafana.com/api/dashboards/7589/revisions/5/download' + sha256: cf020e14465626360418e8b5746818c80d77c0301422f3060879fddc099c2151 + grafana_dashboard_789: + url: 'https://grafana.com/api/dashboards/789/revisions/1/download' + sha256: 6a9b4bdc386062287af4f7d56781103a2e45a51813596a65f03c1ae1d4d3e919 + grafana_dashboard_179: + url: 'https://grafana.com/api/dashboards/179/revisions/7/download' + sha256: 8d67350ff74e715fb1463f2406f24a73377357d90344f8200dad9d1b2a8133c2 + grafana_dashboard_6663: + url: 'https://grafana.com/api/dashboards/6663/revisions/1/download' + sha256: d544d88069e1b793ff3d8f6970df641ad9a66217e69b629621e1ecbb2f06aa05 + grafana_dashboard_10991: + url: 'https://grafana.com/api/dashboards/10991/revisions/11/download' + sha256: 66340fa3256d432287cba75ab5177eb058c77afa7d521a75d58099f95b1bff50 +""" + + EXPECTED_VERBOSE_OUTPUT = """ Manifest summary: -------------------------------------------------- Components detected: - kafka -- monitoring - repository Features detected: - filebeat - firewall -- grafana - image-registry - jmx-exporter - kafka - kafka-exporter - node-exporter -- prometheus - repository - zookeeper -------------------------------------------------- """ + + +EXPECTED_VERBOSE_DASHBOARD_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components detected: +- monitoring +- repository + +Features detected: +- filebeat +- firewall +- grafana +- image-registry +- node-exporter +- prometheus +- repository + +Dashboards to download: +- grafana_dashboard_7249 +- grafana_dashboard_315 +- grafana_dashboard_11074 +- grafana_dashboard_405 +- grafana_dashboard_455 +- grafana_dashboard_9628 +- grafana_dashboard_4279 +- grafana_dashboard_1860 +- grafana_dashboard_7589 +- grafana_dashboard_789 +- grafana_dashboard_179 +- grafana_dashboard_6663 +- grafana_dashboard_10991 +-------------------------------------------------- +""" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py index 9e6f1da501..8df60eea65 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py @@ -1,36 +1,4 @@ -INPUT_MANIFEST_FEATURE_MAPPINGS = """ ---- -kind: epiphany-cluster -title: Epiphany cluster Config -provider: any -name: default -specification: - name: new_cluster - admin_user: - name: operations - key_path: /shared/.ssh/epiphany-operations/id_rsa - components: - repository: - count: 1 - kubernetes_master: - count: 0 - kubernetes_node: - count: 0 - logging: - count: 0 - monitoring: - count: 1 - kafka: - count: 2 - postgresql: - count: 0 - load_balancer: - count: 0 - rabbitmq: - count: 0 - opendistro_for_elasticsearch: - count: 0 -version: 2.0.0dev +FEATURE_MAPPINGS = """ --- kind: configuration/feature-mappings title: Feature mapping to roles @@ -126,17 +94,99 @@ """ +INPUT_MANIFEST_FEATURE_MAPPINGS = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 0 + kafka: + count: 2 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opendistro_for_elasticsearch: + count: 0 +version: 2.0.0dev +{FEATURE_MAPPINGS} +""" + + +INPUT_MANIFEST_WITH_DASHBOARDS = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 1 + kafka: + count: 0 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opendistro_for_elasticsearch: + count: 0 +version: 2.0.0dev +{FEATURE_MAPPINGS} +""" + + EXPECTED_FEATURE_MAPPINGS = { - 'detected-components': ['kafka', 'monitoring', 'repository'], + 'detected-components': ['kafka', 'repository'], 'detected-features': ['filebeat', 'firewall', - 'grafana', 'image-registry', 'jmx-exporter', 'kafka', 'kafka-exporter', 'node-exporter', - 'prometheus', 'repository', 'zookeeper'] } + + +EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS = { + 'detected-components': ['monitoring', 'repository'], + 'detected-features': ['filebeat', + 'firewall', + 'grafana', + 'image-registry', + 'node-exporter', + 'prometheus', + 'repository'] +} diff --git a/cli/src/ansible/AnsibleInventoryCreator.py b/cli/src/ansible/AnsibleInventoryCreator.py index 5e62c78cd5..07c580f05a 100644 --- a/cli/src/ansible/AnsibleInventoryCreator.py +++ b/cli/src/ansible/AnsibleInventoryCreator.py @@ -44,11 +44,11 @@ def get_inventory(self): def get_roles_for_feature(self, component_key): features_map = select_single(self.config_docs, lambda x: x.kind == 'configuration/feature-mappings') - return features_map.specification[component_key] + return features_map.specification.mappings[component_key] def get_available_roles(self): features = select_single(self.config_docs, lambda x: x.kind == 'configuration/features') - return features.specification + return features.specification.features def get_enabled_roles(self): roles = self.get_available_roles() diff --git a/cli/src/commands/Apply.py b/cli/src/commands/Apply.py index 863d5ece21..b6d9e24e3b 100644 --- a/cli/src/commands/Apply.py +++ b/cli/src/commands/Apply.py @@ -230,9 +230,9 @@ def assert_no_postgres_nodes_number_change(self): if self.inventory: next_postgres_node_count = 0 prev_postgres_node_count = len(self.inventory.list_hosts(pattern='postgresql')) - postgres_available = [x for x in features.specification if x.name == 'postgresql'] + postgres_available = [x for x in features.specification.features if x.name == 'postgresql'] if postgres_available[0].enabled: - for key, features in feature_mappings.specification.items(): + for key, features in feature_mappings.specification.mappings.items(): if ('postgresql') in features and key in components: next_postgres_node_count = next_postgres_node_count + components[key].count