Bump logging stack versions

CHANGELOG.md

@@ -14,6 +14,9 @@
 
 - Bump `calico` version to 3.17.0 (PR [#2943](https://github.com/scality/metalk8s/pull/2943))
 
+- Bump `fluent-bit` chart to 2.0.1 and `loki` chart to 2.1.0
+  (PR [#2946](https://github.com/scality/metalk8s/pull/2946))
+
 ### Bug fixes
 
 - [#2908](https://github.com/scality/metalk8s/issues/2908) - Make upgrade script

buildchain/buildchain/versions.py

@@ -219,13 +219,13 @@ def _version_prefix(version: str, prefix: str = 'v') -> str:
     ),
     Image(
         name='loki',
-        version='1.5.0',
-        digest='sha256:922b3f412fdd9a8fb01115b6aebf5dac162647ce1c5ee3637ce1e2cff69e097b',
+        version='2.0.0',
+        digest='sha256:77e138f81a8e253f1d0ea5d2dc329a02212ecab3247c87f85f1f2182a0160ccd',
     ),
     Image(
         name='fluent-bit-plugin-loki',
-        version='1.5.0-amd64',
-        digest='sha256:2d0e9b06a2bf894fa91300fa38a185ac44a7bedcb8a2c63b8f6077e5cf80fc4d',
+        version='1.6.0-amd64',
+        digest='sha256:cb1cd95d0fcf76b626623684f0c8b204a9f773443650c7b3d243b96c29ff7020',
     ),
 )
 

charts/fluent-bit.yaml

@@ -51,7 +51,6 @@ deploymentStrategy: RollingUpdate
 
 image:
   repository: '__image__(fluent-bit-plugin-loki)'
-  tag: 1.5.0-amd64
   pullPolicy: IfNotPresent
 
 nameOverride: fluent-bit

charts/fluent-bit/Chart.yaml

@@ -1,15 +1,14 @@
 apiVersion: v1
-appVersion: v1.5.0
+appVersion: v2.0.0
 description: Uses fluent-bit Loki go plugin for gathering logs and sending them to
   Loki
-engine: gotpl
 home: https://grafana.com/loki
-icon: https://github.com/grafana/loki/raw/master/docs/logo.png
+icon: https://raw.githubusercontent.com/grafana/loki/master/docs/sources/logo.png
 kubeVersion: ^1.10.0-0
 maintainers:
 - email: [email protected]
   name: Loki Maintainers
 name: fluent-bit
 sources:
 - https://github.com/grafana/loki
-version: 0.1.5
+version: 2.0.1

charts/fluent-bit/README.md

@@ -83,6 +83,8 @@ For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/RE
 | `config.loglevel`        | the Fluent Bit log level (debug,info,warn,error).                                                  | `warn`                           |
 | `config.lineFormat`      | The line format to use to send a record (json/key_value)                                           | `json`                           |
 | `config.k8sLoggingParser`| Allow Kubernetes Pods to suggest a pre-defined Parser. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/kubernetes#kubernetes-annotations).                                                                                      | `Off`                           |
+| `config.k8sLoggingExclude`| Allow Kubernetes Pods to exclude their logs from the log processor. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/pipeline/filters/kubernetes)                                                                                             | `Off`
+| `config.memBufLimit`     | Override the default  Mem_Buf_Limit [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/administration/backpressure#mem_buf_limit) | `5MB`
 | `config.removeKeys`      | The list of key to remove from each record                                                         | `[removeKeys,stream]`            |
 | `config.labels`          | A set of labels to send for every log                                                              | `'{job="fluent-bit"}'`           |
 | `config.autoKubernetesLabels` | If set to true, it will add all Kubernetes labels to Loki labels                                   | `false`                          |

charts/fluent-bit/templates/configmap.yaml

@@ -24,12 +24,13 @@ data:
         Path           /var/log/containers/*.log
         Parser         docker
         DB             /run/fluent-bit/flb_kube.db
-        Mem_Buf_Limit  5MB
+        Mem_Buf_Limit  {{ .Values.config.memBufLimit }}
     [FILTER]
         Name           kubernetes
         Match          kube.*
         Kube_URL       https://kubernetes.default.svc:443
         Merge_Log On
+        K8S-Logging.Exclude {{ .Values.config.k8sLoggingExclude }}
         K8S-Logging.Parser {{ .Values.config.k8sLoggingParser }}
     [Output]
         Name loki
@@ -41,7 +42,7 @@ data:
         {{- end }}
         TenantID {{ .Values.config.tenantID }}
         BatchWait {{ .Values.config.batchWait }}
-        BatchSize {{ .Values.config.batchSize }}
+        BatchSize {{ int .Values.config.batchSize }}
         Labels {{ .Values.config.labels }}
         RemoveKeys {{ include "helm-toolkit.utils.joinListWithComma" .Values.config.removeKeys }}
         AutoKubernetesLabels {{ .Values.config.autoKubernetesLabels }}

charts/fluent-bit/templates/daemonset.yaml

@@ -35,9 +35,15 @@ spec:
         {{- end }}
     spec:
       serviceAccountName: {{ template "fluent-bit-loki.serviceAccountName" . }}
-    {{- if .Values.priorityClassName }}
+      {{- if .Values.priorityClassName }}
       priorityClassName: {{ .Values.priorityClassName }}
-    {{- end }}
+      {{- end }}
+      {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.image.pullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
       containers:
         - name: fluent-bit-loki
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"

charts/fluent-bit/values.yaml

@@ -10,10 +10,12 @@ config:
   port: 2020
   tenantID: '""'
   batchWait: 1
-  batchSize: 10240
+  batchSize: 1048576
   loglevel: warn
   lineFormat: json
+  k8sLoggingExclude: "Off"
   k8sLoggingParser: "Off"
+  memBufLimit: "5MB"
   removeKeys:
     - kubernetes
     - stream
@@ -48,8 +50,13 @@ deploymentStrategy: RollingUpdate
 
 image:
   repository: grafana/fluent-bit-plugin-loki
-  tag: 1.5.0-amd64
+  tag: 1.6.0-amd64
   pullPolicy: IfNotPresent
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be present in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  # pullSecrets:
+  #   - myRegistrKeySecretName
 
 nameOverride: fluent-bit-loki
 

charts/loki.yaml

@@ -1,6 +1,5 @@
 image:
   repository: '__image__(loki)'
-  tag: 1.5.0
   pullPolicy: IfNotPresent
 
   ## Optionally specify an array of imagePullSecrets.

charts/loki/Chart.yaml

@@ -1,14 +1,13 @@
 apiVersion: v1
-appVersion: v1.5.0
+appVersion: v2.0.0
 description: 'Loki: like Prometheus, but for logs.'
-engine: gotpl
 home: https://grafana.com/loki
-icon: https://github.com/grafana/loki/raw/master/docs/logo.png
+icon: https://raw.githubusercontent.com/grafana/loki/master/docs/sources/logo.png
 kubeVersion: ^1.10.0-0
 maintainers:
 - email: [email protected]
   name: Loki Maintainers
 name: loki
 sources:
 - https://github.com/grafana/loki
-version: 0.30.2
+version: 2.1.0

charts/loki/README.md

@@ -60,3 +60,8 @@ tls:
     - {{ .Values.ingress.host }}
 ```
 
+## Use Loki Alerting
+
+You can add your own alerting rules with `alerting_groups` in `values.yaml`. This will create a ConfigMap with your rules and additional volumes and mounts for Loki.
+
+This does **not** enable the Loki `ruler` component which does the evaluation of your rules. The `values.yaml` file does contain a simple example. For more details take a look at the official [alerting docs](https://grafana.com/docs/loki/latest/alerting/).

charts/loki/templates/service-headless.yaml

@@ -15,7 +15,7 @@ spec:
     - port: {{ .Values.service.port }}
       protocol: TCP
       name: http-metrics
-      targetPort: http-metrics
+      targetPort: {{ .Values.service.targetPort }}
 {{- if .Values.extraPorts }}
 {{ toYaml .Values.extraPorts | indent 4}}
 {{- end }}

charts/loki/templates/service.yaml

@@ -31,7 +31,7 @@ spec:
     - port: {{ .Values.service.port }}
       protocol: TCP
       name: http-metrics
-      targetPort: http-metrics
+      targetPort: {{ .Values.service.targetPort }}
 {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }}
       nodePort: {{ .Values.service.nodePort }}
 {{- end }}

charts/loki/templates/statefulset.yaml

@@ -67,6 +67,12 @@ spec:
             - name: storage
               mountPath: "/data"
               subPath: {{ .Values.persistence.subPath }}
+            {{- if gt (len .Values.alerting_groups) 0 }}
+            - name: scratch
+              mountPath: /tmp/scratch
+            - name: rules
+              mountPath: /rules
+            {{- end }}
           ports:
             - name: http-metrics
               containerPort: {{ .Values.config.server.http_listen_port }}
@@ -98,6 +104,13 @@ spec:
         {{- toYaml .Values.tolerations | nindent 8 }}
       terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       volumes:
+        {{- if gt (len .Values.alerting_groups) 0 }}
+        - name: scratch
+          emptyDir: {}
+        - name: rules
+          configMap:
+            name: {{ template "loki.fullname" . }}-alerting-rules
+        {{- end }}
         - name: config
           secret:
             secretName: {{ template "loki.fullname" . }}
@@ -129,4 +142,3 @@ spec:
         {{- toYaml .Values.persistence.selector | nindent 8 }}
       {{- end }}
   {{- end }}
-

charts/loki/values.yaml

@@ -1,6 +1,6 @@
 image:
   repository: grafana/loki
-  tag: 1.5.0
+  tag: 2.0.0
   pullPolicy: IfNotPresent
 
   ## Optionally specify an array of imagePullSecrets.
@@ -71,25 +71,44 @@ config:
     reject_old_samples_max_age: 168h
   schema_config:
     configs:
-    - from: 2018-04-15
-      store: boltdb
+    - from: 2020-10-24
+      store: boltdb-shipper
       object_store: filesystem
-      schema: v9
+      schema: v11
       index:
         prefix: index_
-        period: 168h
+        period: 24h
   server:
     http_listen_port: 3100
   storage_config:
-    boltdb:
-      directory: /data/loki/index
+    boltdb_shipper:
+      active_index_directory: /data/loki/boltdb-shipper-active
+      cache_location: /data/loki/boltdb-shipper-cache
+      cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+      shared_store: filesystem
     filesystem:
       directory: /data/loki/chunks
   chunk_store_config:
     max_look_back_period: 0s
   table_manager:
     retention_deletes_enabled: false
     retention_period: 0s
+  compactor:
+    working_directory: /data/loki/boltdb-shipper-compactor
+    shared_store: filesystem
+# Needed for Alerting: https://grafana.com/docs/loki/latest/alerting/
+# This is just a simple example, for more details: https://grafana.com/docs/loki/latest/configuration/#ruler_config
+#  ruler:
+#    storage:
+#      type: local
+#      local:
+#        directory: /rules
+#    rule_path: /tmp/scratch
+#    alertmanager_url: http://alertmanager.svc.namespace:9093
+#    ring:
+#      kvstore:
+#        store: inmemory
+#    enable_api: true
 
 ## Additional Loki container arguments, e.g. log level (debug, info, warn, error)
 extraArgs: {}
@@ -122,6 +141,9 @@ persistence:
   - ReadWriteOnce
   size: 10Gi
   annotations: {}
+  # selector:
+  #   matchLabels:
+  #     app.kubernetes.io/name: loki
   # subPath: ""
   # existingClaim:
 
@@ -170,6 +192,7 @@ service:
   port: 3100
   annotations: {}
   labels: {}
+  targetPort: http-metrics
 
 serviceAccount:
   create: true
@@ -239,3 +262,12 @@ extraPorts: []
 
 # Extra env variables to pass to the loki container
 env: []
+
+# Specify Loki Alerting rules based on this documentation: https://grafana.com/docs/loki/latest/alerting/
+# When specified, you also need to add a ruler config section above. An example is shown in the alerting docs.
+alerting_groups: []
+#  - name: example
+#    rules:
+#    - alert: HighThroughputLogStreams
+#      expr: sum by(container) (rate({job=~"loki-dev/.*"}[1m])) > 1000
+#      for: 2m