From b7ecd4b8a9b9ab5e8d794af2a2312963f62804cb Mon Sep 17 00:00:00 2001 From: Ricky Grassmuck Date: Mon, 25 Oct 2021 13:07:57 -0500 Subject: [PATCH] Refactor based on PR suggestions --- packs/promtail/README.md | 17 +- packs/promtail/metadata.hcl | 2 +- packs/promtail/templates/_helpers.tpl | 125 ++++++++++--- packs/promtail/templates/promtail.nomad.tpl | 89 +++------ packs/promtail/variables.hcl | 195 ++++++++++++++------ 5 files changed, 272 insertions(+), 156 deletions(-) diff --git a/packs/promtail/README.md b/packs/promtail/README.md index cac4e8d..64bfc43 100644 --- a/packs/promtail/README.md +++ b/packs/promtail/README.md @@ -21,17 +21,18 @@ If no custom configuration file is provided, a default template will be used whi |------|-------------|------|---------|:--------:| | [job\_name](#input\_job\_name) | The name to use as the job name which overrides using the pack name. | `string` | `""` | no | | [service\_name](#input\_service\_name) | Name used to register the Consul Service | `string` | `"promtail"` | no | -| [service\_check\_name](#input\_service\_check\_name) | Name of the service check registered with the Consul Service | `string` | `"Readiness"` | no | +| [namespace](#input\_namespace) | The namespace where the job should be placed. | `string` | `"default"` | no | | [datacenters](#input\_datacenters) | A list of datacenters in the region which are eligible for task placement. | `list(string)` | 
[
  "dc1"
]
| no | | [region](#input\_region) | The region where the job should be placed. | `string` | `"global"` | no | | [version\_tag](#input\_version\_tag) | The docker image version. For options, see https://hub.docker.com/grafana/promtail | `string` | `"latest"` | no | -| [http\_port](#input\_http\_port) | The Nomad client port that routes to the Promtail. | `number` | `9080` | no | -| [resources](#input\_resources) | The resource to assign to the promtail service task. | 
object({
    cpu    = number
    memory = number
  })
| 
{
  "cpu": 200,
  "memory": 256
}
| no | +| [privileged](#input\_privileged) | Controls whether the container will be run as a privileged container | `bool` | `false` | no | | [config\_file](#input\_config\_file) | Path to custom Promtail configuration file. | `string` | `""` | no | -| [mount\_journal](#input\_mount\_journal) | Controls whether /var/log/journal is mounted in the container. If true, container will be run privileged. | `bool` | `true` | no | -| [mount\_machine\_id](#input\_mount\_machine\_id) | Controls whether /etc/machine-id is mounted in the container. If true, container will be run privileged. | `bool` | `true` | no | -| [privileged\_container](#input\_privileged\_container) | Run as a privileged container. Setting mount\_journal or mount\_machine\_id to true will override this. | `bool` | `false` | no | | [client\_urls](#input\_client\_urls) | A list of client url's for promtail to send it's data to. | `list(string)` | `[]` | no | | [journal\_max\_age](#input\_journal\_max\_age) | Maximum age of journald entries to scrape. | `string` | `"12h"` | no | -| [log\_level](#input\_log\_level) | Promtail log level configuration. | `string` | `"info"` | no | -| [upstreams](#input\_upstreams) | Define Connect Upstreams used by Promtail. | 
list(object({
    name = string
    port = number
  }))
| `[]` | no | +| [constraints](#input\_constraints) | Constraints to apply to the entire job. | 
list(object({
    attribute = string
    operator  = string
    value     = string
  }))
| 
[
  {
    "attribute": "${attr.kernel.name}",
    "operator": "",
    "value": "linux"
  }
]
| no | +| [promtail\_group\_network](#input\_promtail\_group\_network) | The Promtail network configuration options. | 
object({
    mode  = string
    ports = map(number)
  })
| 
{
  "mode": "bridge",
  "ports": {
    "http": 9090
  }
}
| no | +| [promtail\_group\_services](#input\_promtail\_group\_services) | Configuration options of the promtail services and checks. | 
list(object({
    service_port_label = string
    service_name       = string
    service_tags       = list(string)
    check_enabled      = bool
    check_path         = string
    check_interval     = string
    check_timeout      = string
    upstreams = list(object({
      name = string
      port = number
    }))
  }))
| 
[
  {
    "check_enabled": true,
    "check_interval": "3s",
    "check_path": "/ready",
    "check_timeout": "1s",
    "service_name": "promtail",
    "service_port_label": "http",
    "service_tags": [],
    "upstreams": []
  }
]
| no | +| [resources](#input\_resources) | The resource to assign to the promtail service task. | 
object({
    cpu    = number
    memory = number
  })
| 
{
  "cpu": 200,
  "memory": 256
}
| no | +| [container\_args](#input\_container\_args) | Arguments passed to the Promtail docker container | `list(string)` | 
[
  "-config.file=/etc/promtail/promtail-config.yaml",
  "-log.level=info"
]
| no | +| [extra\_mounts](#input\_extra\_mounts) | Additional mounts to create in the Promtail container | 
list(object({
    type     = string
    source   = string
    target   = string
    readonly = bool
    bind_options = list(object({
      name  = string
      value = string
    }))
  }))
| `[]` | no | +| [default\_mounts](#input\_default\_mounts) | Mounts that are configured when using the default Promtail configuration | 
list(object({
    type     = string
    source   = string
    target   = string
    readonly = bool
    bind_options = list(object({
      name  = string
      value = string
    }))
  }))
| 
[
  {
    "bind_options": [
      {
        "name": "propagation",
        "value": "rshared"
      }
    ],
    "readonly": true,
    "source": "/var/log/journal",
    "target": "/var/log/journal",
    "type": "bind"
  },
  {
    "bind_options": [
      {
        "name": "propagation",
        "value": "rshared"
      }
    ],
    "readonly": false,
    "source": "/etc/machine-id",
    "target": "/etc/machine-id",
    "type": "bind"
  }
]
| no | diff --git a/packs/promtail/metadata.hcl b/packs/promtail/metadata.hcl index 5af39af..e45d0a1 100644 --- a/packs/promtail/metadata.hcl +++ b/packs/promtail/metadata.hcl @@ -1,6 +1,6 @@ app { url = "https://grafana.com/docs/promtail/latest/clients/promtail" - author = "Ricky Grassmuck" + author = "Grafana" } pack { diff --git a/packs/promtail/templates/_helpers.tpl b/packs/promtail/templates/_helpers.tpl index fe9e7de..ab753e2 100644 --- a/packs/promtail/templates/_helpers.tpl +++ b/packs/promtail/templates/_helpers.tpl @@ -16,36 +16,105 @@ region = [[ .promtail.region | quote]] [[- end -]] [[- end -]] +[[- define "constraints" -]] +[[- range $idx, $constraint := . ]] + constraint { + attribute = [[ $constraint.attribute | quote ]] + [[ if $constraint.operator -]] + operator = [[ $constraint.operator | quote ]] + [[ end -]] + value = [[ $constraint.value | quote ]] + } +[[- end ]] +[[- end -]] + // Use provided config file if defined, else render a default [[- define "promtail_config" -]] -[[- if (eq .promtail.config_file "") -]] -server: - http_listen_port: [[ .promtail.http_port ]] - log_level: [[ .promtail.log_level ]] - -positions: - filename: /tmp/positions.yaml - -clients: - [[ range $idx, $url := .promtail.client_urls ]][[if $idx]] [[end]][[ $url | printf "- url: %s\n" ]][[ end ]] -scrape_configs: -- job_name: journal - journal: - max_age: [[ .promtail.journal_max_age ]] - json: false - labels: - job: systemd-journal - relabel_configs: - - source_labels: - - __journal__systemd_unit - target_label: systemd_unit - - source_labels: - - __journal__hostname - target_label: nodename - - source_labels: - - __journal_syslog_identifier - target_label: syslog_identifier -[[- else -]] +[[- if (eq .promtail.config_file "") ]] + server: + http_listen_port: [[ .promtail.http_port ]] + log_level: [[ .promtail.log_level ]] + + positions: + filename: /tmp/positions.yaml + + clients: + [[ range $idx, $url := .promtail.client_urls ]][[if $idx]] [[end]][[ $url | printf "- url: %s\n" ]][[ end ]] + scrape_configs: + - job_name: journal + journal: + max_age: [[ .promtail.journal_max_age ]] + json: false + labels: + job: systemd-journal + relabel_configs: + - source_labels: + - __journal__systemd_unit + target_label: systemd_unit + - source_labels: + - __journal__hostname + target_label: nodename + - source_labels: + - __journal_syslog_identifier + target_label: syslog_identifier +[[- else ]] [[ $config := .promtail.config_file ]][[ fileContents $config ]] +[[- end ]] [[- end -]] + +// Generic "service" block template +[[- define "service" -]] +[[- range $idx, $service := . ]] + service { + name = [[ $service.service_name | quote ]] + port = [[ $service.service_port_label | quote ]] + tags = [ [[- range $idx, $dc := $service.service_tags ]][[if $idx]],[[end]][[ $dc | quote ]][[ end -]] ] + [[- if gt (len $service.upstreams) 0 ]] + connect { + sidecar_service { + proxy { + [[- if gt (len $service.upstreams) 0 ]] + [[- range $upstream := $service.upstreams ]] + upstreams { + destination_name = [[ $upstream.name | quote ]] + local_bind_port = [[ $upstream.port ]] + } + [[- end ]] + [[- end ]] + } + } + } + [[- end ]] + check { + type = "http" + path = [[ $service.check_path | quote ]] + interval = [[ $service.check_interval | quote ]] + timeout = [[ $service.check_timeout | quote ]] + } + } +[[- end ]] +[[- end -]] + +[[- define "env" -]] + env { + [[- range $idx, $var := . ]] + [[ $var.name | quote ]] = [[ $var.value | quote ]] + [[- end ]] + } +[[- end -]] + +[[- define "mounts" -]] +[[- range $idx, $mount := . ]] + mount { + type = [[ $mount.type | quote ]] + target = [[ $mount.target | quote ]] + source = [[ $mount.source | quote ]] + readonly = [[ $mount.readonly ]] + bind_options { + [[- range $idx, $opt := $mount.bind_options ]] + [[ $opt.name ]] = [[ $opt.value | quote ]] + [[- end ]] + } + } +[[- end ]] [[- end -]] \ No newline at end of file diff --git a/packs/promtail/templates/promtail.nomad.tpl b/packs/promtail/templates/promtail.nomad.tpl index b5bbfdf..1f3c014 100644 --- a/packs/promtail/templates/promtail.nomad.tpl +++ b/packs/promtail/templates/promtail.nomad.tpl @@ -1,70 +1,40 @@ job [[ template "job_name" . ]] { [[ template "region" . ]] - datacenters = [ [[ range $idx, $dc := .promtail.datacenters ]][[if $idx]],[[end]][[ $dc | quote ]][[ end ]] ] - type = "system" + namespace = [[ .promtail.namespace | quote ]] + type = "system" - constraint { - attribute = "${attr.kernel.name}" - value = "linux" - } + [[ template "constraints" .promtail.constraints ]] group "promtail" { - count = 1 - network { - mode = "bridge" - port "http" { - to = [[ .promtail.http_port ]] - } - } - - service { - name = [[ .promtail.service_name | quote ]] - port = "http" - [[- if not (eq (len .promtail.upstreams) 0) ]] // Render Connect upstreams if defined - connect { - sidecar_service { - proxy { - [[- range $upstream := .promtail.upstreams ]] - upstreams { - destination_name = [[ $upstream.name | quote ]] - local_bind_port = [[ $upstream.port ]] - } - [[- end ]] - } - } + mode = [[ .promtail.promtail_group_network.mode | quote ]] + [[- range $label, $to := .promtail.promtail_group_network.ports ]] + port [[ $label | quote ]] { + to = [[ $to ]] } [[- end ]] - check { - name = [[ .promtail.service_check_name | quote ]] - port = "http" - type = "http" - path = "/ready" - timeout = "2s" - interval = "10s" - } } + [[- if .promtail.promtail_task_services ]] + [[ template "service" .promtail.promtail_group_services ]] + [[- end ]] + task "promtail" { driver = "docker" template { destination = "local/promtail-config.yaml" - data = <