libmapstack.jinja

{#- -*- coding: utf-8 -*- #}
{#- vim: ft=jinja #}

{#- Get the `tplroot` from `tpldir` #}
{%- set tplroot = tpldir.split("/")[0] %}
{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map with context %}

{%- set _default_config_dirs = [
      "parameters/",
      tplroot ~ "/parameters"
    ] %}

{%- macro mapstack(
      matchers,
      defaults=None,
      dirs=_default_config_dirs,
      log_prefix="libmapstack: "
    ) %}
{#-
    Load configuration in the order of `matchers` and merge
    successively the values with `defaults`.

    The `matchers` are processed using `libmatchers.jinja` to select
    the configuration sources from where the values are loaded.

    Parameters:

      - `matchers`: list of matchers in the form
        `[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<QUERY>`

      - `defaults`: dictionary of default values to start the merging,
        they are considered built-ins. It must conform to the same
        layout as the YAML files: a mandatory `values` key and two
        optional `strategy` and `merge_lists` keys.

      - `dirs`: list of directory where to look-up the configuration
        file matching the matchers, by default a global `salt://parameters/`
        and a per formula `salt://<tplroot>/parameters`

      - `log_prefix`: prefix used in the log outputs, by default it is
        `libmapstack: `

    Example: On a Debian system with `roles=["nginx/server", "telegraf"]`

      {%- set settings = mapstack(
            matchers=[
              "Y:G@os_family",
              "I@" ~ tplroot,
              "Y:C@roles",
            ],
            dirs=["defaults", tplroot ~ "/parameters"],
          )
          | load_yaml %}

      This will merge the values:

        - starting with the default empty dictionary `{}` (no
          `defaults` parameter)

        - from the YAML files

          - `salt://defaults/os_family/Debian.yaml`

          - `salt://{{ tplroot }}/parameters/os_family/Debian.yaml`

        - from the pillar `salt["pillar.get"](tplroot)`

        - from the `nginx/server` YAML files:

          - `salt://defaults/roles/nginx/server.yaml`

          - `salt://{{ tplroot }}/parameters/roles/nginx/server.yaml`

        - from the `telegraf` YAML files:

          - `salt://defaults/roles/telegraf.yaml`

          - `salt://{{ tplroot }}/parameters/roles/telegraf.yaml`

    Each YAML file and the `defaults` parameters must conform to the
    following layout:

      - a mandatory `values` key to store the configuration values

      - two optional keys to configure the use of `salt.slsutil.merge`

        - an optional `strategy` key to configure the merging
          strategy, for example `strategy: 'recurse'`, the default is
          `smart`

        - an optional `merge_lists` key to configure if lists should
          be merged or overridden for the `recurse` and `overwrite`
          strategies, for example `merge_lists: 'true'`
#}
{%-   set stack = defaults | default({"values": {} }, boolean=True) %}

{#-   Build configuration file names based on matchers #}
{%-   set config_get_strategy = salt["config.get"](tplroot ~ ":strategy", None) %}
{%-   set matchers = parse_matchers(
        matchers,
        config_get_strategy=config_get_strategy,
        log_prefix=log_prefix
      )
      | load_yaml %}

{#{%-   do salt["log.debug"](
        log_prefix
        ~ "built-in configuration:\n"
        ~ {"values": defaults | traverse("values")}
        | yaml(False)
      ) %}#}

{%-   for param_dir in dirs %}
{%-     for matcher in matchers %}
{#-       `slsutil.merge` options from #}
{#-       1. the `value` #}
{#-       2. the `defaults` #}
{#-       3. the built-in #}
{%-       set strategy = matcher.value
          | traverse(
            "strategy",
            defaults
            | traverse(
              "strategy",
              "smart"
            )
          ) %}
{%-       set merge_lists = matcher.value
          | traverse(
            "merge_lists",
            defaults
            | traverse(
              "merge_lists",
              False
            )
          )
          | to_bool %}

{%-       if matcher.type in query_map.keys() %}
{#-         No value is an empty list, must be a dict for `stack.update` #}
{%-         set normalized_value = matcher.value | default({}, boolean=True) %}

{#-         Merge in `mapdata.<query>` instead of directly in `mapdata` #}
{%-         set is_sub_key = matcher.option | default(False) == "SUB" %}
{%-         if is_sub_key %}
{#-           Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
{%-           set value = { matcher.query | regex_replace(":lookup$", ""): normalized_value } %}
{%-         else %}
{%-           set value = normalized_value %}
{%-         endif %}

{#{%-         do salt["log.debug"](
              log_prefix
              ~ "merge "
              ~ "sub key " * is_sub_key
              ~ "'"
              ~ matcher.query
              ~ "' retrieved with '"
              ~ matcher.query_method
              ~ "', merge: strategy='"
              ~ strategy
              ~ "', lists='"
              ~ merge_lists
              ~ "':\n"
              ~ value
              | yaml(False)
            ) %}#}

{%-         do stack.update(
              {
                "values": salt["slsutil.merge"](
                  stack["values"],
                  value,
                  strategy=strategy,
                  merge_lists=merge_lists,
                )
              }
            ) %}

{%-       else %}
{#-         Load YAML file matching the grain/pillar/... #}
{#-         Fallback to use the source name as a direct filename #}

{%-         if matcher.value is sequence and matcher.value | length == 0 %}
{#-           Mangle `matcher.value` to use it as literal path #}
{%-           set query_parts = matcher.query.split("/") %}
{%-           set yaml_dirname = query_parts[0:-1] | join("/") %}
{%-           set yaml_names = query_parts[-1] %}
{%-         else %}
{%-           set yaml_dirname = matcher.query %}
{%-           set yaml_names = matcher.value %}
{%-         endif %}

{#-         Some configuration return list #}
{%-         if yaml_names is string %}
{%-           set yaml_names = [yaml_names] %}
{%-         elif yaml_names is sequence %}
{#-           Convert to strings if it's a sequence of numbers #}
{%-           set yaml_names = yaml_names | map("string") | list %}
{%-         else %}
{%-           set yaml_names = [yaml_names | string] %}
{%-         endif %}

{#-         Try to load a `.yaml.jinja` file for each `.yaml` file #}
{%-         set all_yaml_names = [] %}
{%-         for name in yaml_names %}
{%-           set extension = name.rpartition(".")[2] %}
{%-           if extension not in ["yaml", "jinja"] %}
{%-             do all_yaml_names.extend([name ~ ".yaml", name ~ ".yaml.jinja"]) %}
{%-           elif extension == "yaml" %}
{%-             do all_yaml_names.extend([name, name ~ ".jinja"]) %}
{%-           else %}
{%-             do all_yaml_names.append(name) %}
{%-           endif %}
{%-         endfor %}

{#-         `yaml_dirname` can be an empty string with literal path like `myconf.yaml` #}
{%-         set yaml_dir = [
              param_dir,
              yaml_dirname
            ]
            | select
            | join("/") %}

{%-         for yaml_name in all_yaml_names %}
{%-           set yaml_filename = [
                yaml_dir.rstrip("/"),
                yaml_name
              ]
              | select
              | join("/") %}

{#{%-           do salt["log.debug"](
                log_prefix
                ~ "load configuration values from "
                ~ yaml_filename
              ) %}#}
{%-           load_yaml as yaml_values %}
{%-             include yaml_filename ignore missing %}
{%-           endload %}

{%-           if yaml_values %}
{#{%-             do salt["log.debug"](
                  log_prefix
                  ~ "loaded configuration values from "
                  ~ yaml_filename
                  ~ ":\n"
                  ~ yaml_values
                  | yaml(False)
                ) %}#}

{#-             `slsutil.merge` options from #}
{#-             1. the `value` #}
{#-             2. the `defaults` #}
{#-             3. the built-in #}
{%-             set strategy = yaml_values
                  | traverse(
                    "strategy",
                    defaults
                    | traverse(
                      "strategy",
                      "smart"
                    )
                  ) %}
{%-             set merge_lists = yaml_values
                  | traverse(
                    "merge_lists",
                    defaults
                    | traverse(
                      "merge_lists",
                      False
                    )
                  )
                  | to_bool %}
{%-             do stack.update(
                  {
                    "values": salt["slsutil.merge"](
                      stack["values"],
                      yaml_values
                      | traverse("values", {}),
                      strategy=strategy,
                      merge_lists=merge_lists,
                    )
                  }
                ) %}
{#{%-             do salt["log.debug"](
                  log_prefix
                  ~ "merged configuration values from "
                  ~ yaml_filename
                  ~ ", merge: strategy='"
                  ~ strategy
                  ~ "', merge_lists='"
                  ~ merge_lists
                  ~ "':\n"
                  ~ {"values": stack["values"]}
                  | yaml(False)
                ) %}#}
{%-           endif %}
{%-         endfor %}
{%-       endif %}
{%-     endfor %}
{%-   endfor %}

{#{%-   do salt["log.debug"](
        log_prefix
        ~ "final configuration values:\n"
        ~ {"values": stack["values"]}
        | yaml(False)
      ) %}#}

{#- Output stack as YAML, caller should use with something like #}
{#- `{%- set config = mapstack(matchers=["foo"]) | load_yaml %}` #}
{{ stack | yaml }}

{%- endmacro %}