From dc751ab6c16042c89dc2e7cff1eaa93d682d75aa Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Thu, 30 Apr 2020 12:29:28 +0200 Subject: [PATCH 1/7] salt: render utility should import csc defaults Since we make use of csc defaults in the charts, the render utility script should import the csc defaults using jinja syntax --- charts/render.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/charts/render.py b/charts/render.py index a1210ecaf5..b6d3133a66 100755 --- a/charts/render.py +++ b/charts/render.py @@ -41,6 +41,7 @@ #!jinja | metalk8s_kubernetes {{%- from "metalk8s/repo/macro.sls" import build_image_name with context %}} +{csc_defaults} {configlines} {{% raw %}} @@ -212,15 +213,27 @@ def main(): doc=doc ) ) + if args.service_configs: + import_csc_yaml = '\n'.join( + ("{{% import_yaml 'metalk8s/addons/{0}/config/{1}.yaml' as " + "{1}_defaults with context %}}").format( + args.name, service_config[0] + ) for service_config in args.service_configs + ) - config = '\n'.join( - ("{{%- set {} = salt.metalk8s_service_configuration" - ".get_service_conf('{}', '{}') %}}").format( - service_config[0], args.namespace, service_config[1] - ) for service_config in args.service_configs - ) if args.service_configs else '' + config = '\n'.join( + ("{{%- set {0} = salt.metalk8s_service_configuration" + ".get_service_conf('{1}', '{2}', {0}_defaults) %}}").format( + service_config[0], args.namespace, service_config[1] + ) for service_config in args.service_configs + ) + else: + import_csc_yaml = '' + config = '' - sys.stdout.write(START_BLOCK.format(configlines=config).lstrip()) + sys.stdout.write(START_BLOCK.format( + csc_defaults=import_csc_yaml, configlines=config).lstrip() + ) sys.stdout.write('\n') stream = io.StringIO() From 020387cd623e5af95de3f52d3082849a44e6a55e Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Thu, 30 Apr 2020 12:31:39 +0200 Subject: [PATCH 2/7] kubernetes: create empty service configuration configmaps This commit refactors service configuration configmaps to become empty objects that will hold and persist user customizable settings --- .../dex/deployed/service-configuration.sls | 13 +------- .../deployed/service-configuration.sls | 32 +++---------------- 2 files changed, 6 insertions(+), 39 deletions(-) diff --git a/salt/metalk8s/addons/dex/deployed/service-configuration.sls b/salt/metalk8s/addons/dex/deployed/service-configuration.sls index da7cf068db..d42dd49b00 100644 --- a/salt/metalk8s/addons/dex/deployed/service-configuration.sls +++ b/salt/metalk8s/addons/dex/deployed/service-configuration.sls @@ -23,18 +23,7 @@ Create dex-config ConfigMap: config.yaml: |- apiVersion: addons.metalk8s.scality.com kind: DexConfig - spec: - deployment: - replicas: 2 - localuserstore: - enabled: true - userlist: - - email: "admin@metalk8s.invalid" - hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" - username: "admin" - userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" - connectors: [] - + spec: {} {%- else %} diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/service-configuration.sls b/salt/metalk8s/addons/prometheus-operator/deployed/service-configuration.sls index 2dee4540e3..b5389805a9 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/service-configuration.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/service-configuration.sls @@ -39,9 +39,8 @@ Create grafana-config ConfigMap: config.yaml: |- apiVersion: addons.metalk8s.scality.com kind: GrafanaConfig - spec: - deployment: - replicas: 1 + spec: {} + {%- else %} metalk8s-grafana-config ConfigMap already exist: @@ -63,9 +62,8 @@ Create prometheus-config ConfigMap: config.yaml: |- apiVersion: addons.metalk8s.scality.com kind: PrometheusConfig - spec: - deployment: - replicas: 1 + spec: {} + {%- else %} metalk8s-prometheus-config ConfigMap already exist: @@ -87,27 +85,7 @@ Create alertmanager-config ConfigMap: config.yaml: |- apiVersion: addons.metalk8s.scality.com kind: AlertmanagerConfig - spec: - deployment: - replicas: 1 - notification: - config: - global: - resolve_timeout: 5m - templates: [] - route: - group_by: ['job'] - group_wait: 30s - group_interval: 5m - repeat_interval: 12h - receiver: 'null' - routes: - - match: - alertname: Watchdog - receiver: 'null' - receivers: - - name: 'null' - inhibit_rules: [] + spec: {} {%- else %} From 272b69ccce19c2e0f3d68e879b1c0de5f06a796e Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Thu, 30 Apr 2020 12:34:34 +0200 Subject: [PATCH 3/7] charts: render the prometheus-operator charts with default service configurations This chart is re-rendered using: ``` $ ./charts/render.py prometheus-operator --namespace metalk8s-monitoring charts/prometheus-operator.yaml --service-config grafana metalk8s-grafana-config --service-config prometheus metalk8s-prometheus-config --service-config alertmanager metalk8s-alertmanager-config charts/prometheus-operator/ > salt/metalk8s/addons/prometheus-operator/deployed/chart.sls ``` --- .../deployed/alertmanager-configuration-secret.sls | 4 +++- .../addons/prometheus-operator/deployed/chart.sls | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/alertmanager-configuration-secret.sls b/salt/metalk8s/addons/prometheus-operator/deployed/alertmanager-configuration-secret.sls index b6a0f98cdf..36e7a5ed54 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/alertmanager-configuration-secret.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/alertmanager-configuration-secret.sls @@ -1,5 +1,7 @@ +{% import_yaml 'metalk8s/addons/prometheus-operator/config/alertmanager.yaml' as alertmanager_defaults with context %} + {%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf( - 'metalk8s-monitoring', 'metalk8s-alertmanager-config' + 'metalk8s-monitoring', 'metalk8s-alertmanager-config', alertmanager_defaults ) %} diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/chart.sls b/salt/metalk8s/addons/prometheus-operator/deployed/chart.sls index e8c7b9ece4..69386e3bd2 100644 --- a/salt/metalk8s/addons/prometheus-operator/deployed/chart.sls +++ b/salt/metalk8s/addons/prometheus-operator/deployed/chart.sls @@ -1,9 +1,12 @@ #!jinja | metalk8s_kubernetes {%- from "metalk8s/repo/macro.sls" import build_image_name with context %} -{%- set grafana = salt.metalk8s_service_configuration.get_service_conf('metalk8s-monitoring', 'metalk8s-grafana-config') %} -{%- set prometheus = salt.metalk8s_service_configuration.get_service_conf('metalk8s-monitoring', 'metalk8s-prometheus-config') %} -{%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf('metalk8s-monitoring', 'metalk8s-alertmanager-config') %} +{% import_yaml 'metalk8s/addons/prometheus-operator/config/grafana.yaml' as grafana_defaults with context %} +{% import_yaml 'metalk8s/addons/prometheus-operator/config/prometheus.yaml' as prometheus_defaults with context %} +{% import_yaml 'metalk8s/addons/prometheus-operator/config/alertmanager.yaml' as alertmanager_defaults with context %} +{%- set grafana = salt.metalk8s_service_configuration.get_service_conf('metalk8s-monitoring', 'metalk8s-grafana-config', grafana_defaults) %} +{%- set prometheus = salt.metalk8s_service_configuration.get_service_conf('metalk8s-monitoring', 'metalk8s-prometheus-config', prometheus_defaults) %} +{%- set alertmanager = salt.metalk8s_service_configuration.get_service_conf('metalk8s-monitoring', 'metalk8s-alertmanager-config', alertmanager_defaults) %} {% raw %} @@ -40382,7 +40385,7 @@ spec: template: metadata: annotations: - checksum/config: 7bdc765fb193e66e7a094a5fb0ffe34218370368d820cffbaf93cae218ebe61f + checksum/config: 822028727d07edab6b024280685ef2510fcbae564ca11bea646dde763fd9559c checksum/dashboards-json-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b checksum/sc-dashboard-provider-config: 424200eb6040b7b1ec58add370935c65963d856f8ece2725caaa8390a3b54eee checksum/secret: 90b18138547156baaa1588680e3842aabcb31605c7f7fb8baa7eacc2b6d4822b From e21326e51fa474c04f5575554758493cbc53ed6b Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Thu, 30 Apr 2020 12:39:00 +0200 Subject: [PATCH 4/7] charts: re-render the dex charts to use default service configurations This chart is re-rendered using: ``` $ ./charts/render.py dex --namespace metalk8s-auth charts/dex.yaml --service-config dex metalk8s-dex-config charts/dex/ > salt/metalk8s/addons/dex/deployed/chart.sls ``` --- salt/metalk8s/addons/dex/deployed/chart.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/metalk8s/addons/dex/deployed/chart.sls b/salt/metalk8s/addons/dex/deployed/chart.sls index 4a1c2c360c..0a55910064 100644 --- a/salt/metalk8s/addons/dex/deployed/chart.sls +++ b/salt/metalk8s/addons/dex/deployed/chart.sls @@ -1,7 +1,8 @@ #!jinja | metalk8s_kubernetes {%- from "metalk8s/repo/macro.sls" import build_image_name with context %} -{%- set dex = salt.metalk8s_service_configuration.get_service_conf('metalk8s-auth', 'metalk8s-dex-config') %} +{% import_yaml 'metalk8s/addons/dex/config/dex.yaml' as dex_defaults with context %} +{%- set dex = salt.metalk8s_service_configuration.get_service_conf('metalk8s-auth', 'metalk8s-dex-config', dex_defaults) %} {% raw %} From 17e82856246423b98712e2190e7f6691b9229e76 Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Thu, 30 Apr 2020 12:41:14 +0200 Subject: [PATCH 5/7] salt: implement merge strategy to make `csc` update-able This commit makes use of the salt module `dictupdate.merge` ensuring that default CSC values are merged with user specified CSC while still providing a fallback configuation for extreme scenarios. This above merge is implemented to ensure that we can do the following: 1. Deploy metalk8s addons with minimum default service values 2. Metalk8s administrators can update and add new configurations values 3. In cases of no Administrator configurations, the default values will be used to deploy the services. 4. In cases where CSC configmaps are removed, we can fall back to default values closes: #2488 --- .../metalk8s_service_configuration.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/salt/_modules/metalk8s_service_configuration.py b/salt/_modules/metalk8s_service_configuration.py index 2bcb0c3165..f4fdf0fa1e 100644 --- a/salt/_modules/metalk8s_service_configuration.py +++ b/salt/_modules/metalk8s_service_configuration.py @@ -3,6 +3,7 @@ import logging import yaml +import salt from salt.exceptions import CommandExecutionError @@ -18,6 +19,7 @@ def __virtual__(): def get_service_conf( namespace, configmap_name, + default_csc, apiVersion=None, kind=None, **kwargs @@ -27,6 +29,7 @@ def get_service_conf( Arguments: configmap_name: the ConfigMap name namespace: the Namespace where the ConfigMap is stored + default_csc: the CSC imported from YAML file Returns: A dict of a specific service configuration @@ -35,13 +38,21 @@ def get_service_conf( .. code-block:: bash - salt-call metalk8s_service_configuration.get_service_conf "metalk8s-monitoring" "metalk8s-prometheus-config" + salt-call metalk8s_service_configuration.get_service_conf + "metalk8s-monitoring" "metalk8s-prometheus-config" + '{"apiVersion":"addons.metalk8s.scality.com","kind":"PrometheusConfig","spec":{"deployment":{"replicas":1}}}' """ if not configmap_name: - raise Exception( + raise CommandExecutionError( 'Expected a ConfigMap name but got {}'.format(configmap_name) ) + if not isinstance(default_csc, dict): + raise CommandExecutionError( + 'Expected default CSC for ConfigMap {} but got {}'.format( + configmap_name, default_csc + ) + ) try: manifest = __salt__[ @@ -83,10 +94,6 @@ def get_service_conf( 'Expected `config.yaml` as yaml in the ConfigMap {} but got {}' .format(configmap_name, config) ) - - if not apiVersion and not kind: - return config - if apiVersion and config['apiVersion'] != apiVersion: raise CommandExecutionError( 'Expected value {} for key apiVersion, got {}'.format( @@ -99,6 +106,9 @@ def get_service_conf( kind, config['kind'] ) ) + merged_config = salt.utils.dictupdate.merge( + default_csc, config, strategy='recurse', + merge_lists=True + ) - #Todo: Need a full schema validation of the ConfigMap data portion - return config + return merged_config From a0ceca9b8e6a9c400d9b255ae14322ddc268a3f5 Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Mon, 4 May 2020 02:49:51 +0200 Subject: [PATCH 6/7] csc: add default csc configurations as YAML files This commit adds csc configuration files for dex, alertmanager, prometheus and grafana to their respective addon folders. These default configurations will be read during addon rendering by salt and then the default values will be used to deploy the respective services. --- buildchain/buildchain/salt_tree.py | 4 +++ salt/metalk8s/addons/dex/config/dex.yaml | 16 ++++++++++++ .../config/alertmanager.yaml | 26 +++++++++++++++++++ .../prometheus-operator/config/grafana.yaml | 8 ++++++ .../config/prometheus.yaml | 8 ++++++ 5 files changed, 62 insertions(+) create mode 100644 salt/metalk8s/addons/dex/config/dex.yaml create mode 100644 salt/metalk8s/addons/prometheus-operator/config/alertmanager.yaml create mode 100644 salt/metalk8s/addons/prometheus-operator/config/grafana.yaml create mode 100644 salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index b15c727e03..896aae51cf 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -226,6 +226,7 @@ def _get_parts(self) -> Iterator[str]: Path('salt/metalk8s/addons/dex/ca/advertised.sls'), Path('salt/metalk8s/addons/dex/certs/init.sls'), Path('salt/metalk8s/addons/dex/certs/server.sls'), + Path('salt/metalk8s/addons/dex/config/dex.yaml'), Path('salt/metalk8s/addons/dex/deployed/chart.sls'), Path('salt/metalk8s/addons/dex/deployed/init.sls'), Path('salt/metalk8s/addons/dex/deployed/namespace.sls'), @@ -269,6 +270,9 @@ def _get_parts(self) -> Iterator[str]: Path('salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls'), Path('salt/metalk8s/addons/prometheus-adapter/deployed/init.sls'), + Path('salt/metalk8s/addons/prometheus-operator/config/alertmanager.yaml'), + Path('salt/metalk8s/addons/prometheus-operator/config/grafana.yaml'), + Path('salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml'), Path('salt/metalk8s/addons/prometheus-operator/deployed/' 'alertmanager-configuration-secret.sls'), Path('salt/metalk8s/addons/prometheus-operator/deployed/chart.sls'), diff --git a/salt/metalk8s/addons/dex/config/dex.yaml b/salt/metalk8s/addons/dex/config/dex.yaml new file mode 100644 index 0000000000..3edb3cfed3 --- /dev/null +++ b/salt/metalk8s/addons/dex/config/dex.yaml @@ -0,0 +1,16 @@ +--- +# Configuration of the Dex (OIDC) service +apiVersion: addons.metalk8s.scality.com +kind: DexConfig +spec: + # Configure the Dex Deployment + deployment: + replicas: 2 + localuserstore: + enabled: true + userlist: + - email: "admin@metalk8s.invalid" + hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" + username: "admin" + userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" + connectors: [] diff --git a/salt/metalk8s/addons/prometheus-operator/config/alertmanager.yaml b/salt/metalk8s/addons/prometheus-operator/config/alertmanager.yaml new file mode 100644 index 0000000000..e60976c22c --- /dev/null +++ b/salt/metalk8s/addons/prometheus-operator/config/alertmanager.yaml @@ -0,0 +1,26 @@ +--- +# Configuration of the Alertmanager service +apiVersion: addons.metalk8s.scality.com +kind: AlertmanagerConfig +spec: + # Configure the Alertmanager Deployment + deployment: + replicas: 1 + notification: + config: + global: + resolve_timeout: 5m + templates: [] + route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - match: + alertname: Watchdog + receiver: 'null' + receivers: + - name: 'null' + inhibit_rules: [] diff --git a/salt/metalk8s/addons/prometheus-operator/config/grafana.yaml b/salt/metalk8s/addons/prometheus-operator/config/grafana.yaml new file mode 100644 index 0000000000..e2593814be --- /dev/null +++ b/salt/metalk8s/addons/prometheus-operator/config/grafana.yaml @@ -0,0 +1,8 @@ +--- +# Configuration of the Grafana service +apiVersion: addons.metalk8s.scality.com +kind: GrafanaConfig +spec: + # Configure the Grafana Deployment + deployment: + replicas: 1 diff --git a/salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml b/salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml new file mode 100644 index 0000000000..34a0823602 --- /dev/null +++ b/salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml @@ -0,0 +1,8 @@ +--- +# Configuration of the Prometheus service +apiVersion: addons.metalk8s.scality.com +kind: PrometheusConfig +spec: + # Configure the Prometheus Deployment + deployment: + replicas: 1 From e12ce230dd1f4ca3f2f3b58869a0f6b5e12b3541 Mon Sep 17 00:00:00 2001 From: Claude Ebaneck Date: Tue, 5 May 2020 00:10:41 +0200 Subject: [PATCH 7/7] test: refactor csc tests to handle empty csc ConfigMaps Initially, we expect default CSC ConfigMaps to contain default values and so we read these default values e.g `replicas` to perform csc persistence tests But with our new approach, default CSC ConfigMaps are deployed without default service values, so we need to adapt the test accordingly --- .../features/service_configuration.feature | 2 +- .../post/steps/test_service_configuration.py | 32 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/post/features/service_configuration.feature b/tests/post/features/service_configuration.feature index c5928daf5e..fdb24decac 100644 --- a/tests/post/features/service_configuration.feature +++ b/tests/post/features/service_configuration.feature @@ -4,7 +4,7 @@ Feature: Cluster and Services Configurations Given the Kubernetes API is available And pods with label 'app.kubernetes.io/name=dex' are 'Ready' And we have 2 running pod labeled 'app.kubernetes.io/name=dex' in namespace 'metalk8s-auth' - And we have a 'metalk8s-dex-config' CSC in namespace 'metalk8s-auth' with 'spec.deployment.replicas' equal to '2' + And we have a 'metalk8s-dex-config' CSC in namespace 'metalk8s-auth' When we update 'metalk8s-dex-config' CSC in namespace 'metalk8s-auth' 'spec.deployment.replicas' to '3' And we apply the 'metalk8s.addons.dex.deployed' salt state Then we have '3' at 'status.available_replicas' for 'dex' Deployment in namespace 'metalk8s-auth' diff --git a/tests/post/steps/test_service_configuration.py b/tests/post/steps/test_service_configuration.py index aaf6080d67..72c9f2b21c 100644 --- a/tests/post/steps/test_service_configuration.py +++ b/tests/post/steps/test_service_configuration.py @@ -11,6 +11,20 @@ from tests import utils +# Constants {{{ + + +DEFAULT_CSC_CONFIG_YAML = """ +apiVersion: addons.metalk8s.scality.com +kind: DexConfig +spec: + deployment: + replicas: {replicas} +""" + + +# }}} + # Fixtures {{{ @@ -42,15 +56,12 @@ def test_service_config_propagation(host): @given(parsers.parse( - "we have a '{name}' CSC in namespace '{namespace}' with " - "'{path}' equal to '{value}'")) + "we have a '{name}' CSC in namespace '{namespace}'")) def check_csc_configuration( k8s_client, csc, name, namespace, - path, - value ): csc_response = csc.get(name, namespace) @@ -61,12 +72,6 @@ def check_csc_configuration( ) csc_obj = csc.load(csc_response, name, namespace) - response_value = utils.get_dict_element(csc_obj, path) - - assert literal_eval(value) == response_value, ( - "Expected value {} for key {} in ConfigMap {} found in namespace {}, " - "got {}".format(value, path, name, namespace, response_value) - ) return dict(csc_obj=csc_obj) @@ -88,14 +93,13 @@ def update_service_configuration( value ): - full_csc = csc.get(name, namespace) - csc_obj = utils.set_dict_element( - csc.load(full_csc, name, namespace), path, literal_eval(value) + csc_from_yaml = yaml.safe_load( + DEFAULT_CSC_CONFIG_YAML.format(replicas=value) ) patch = { 'data': { 'config.yaml': yaml.safe_dump( - csc_obj, default_flow_style=False + csc_from_yaml, default_flow_style=False ) } }