From 934a3bc4492401d128fa1972c404d033cd79659d Mon Sep 17 00:00:00 2001 From: Teddy Andrieux Date: Mon, 28 Nov 2022 18:52:28 +0100 Subject: [PATCH] salt: Add a workaround CronJob to delete old Loki chunks Waiting for a proper fix add a simple CronJob to delete old Loki chunks --- CHANGELOG.md | 8 ++- buildchain/buildchain/salt_tree.py | 3 + charts/loki.yaml | 2 + charts/loki/Chart.yaml | 2 +- charts/loki/README.md | 2 +- charts/loki/docs/examples/README.md | 4 +- charts/loki/templates/_helpers.tpl | 69 +++++++++++++++++++ charts/loki/templates/ingress.yaml | 21 +----- .../loki/templates/read/statefulset-read.yaml | 2 +- .../templates/single-binary/statefulset.yaml | 2 +- .../loki/templates/tokengen/job-tokengen.yaml | 2 +- .../templates/write/statefulset-write.yaml | 2 +- charts/loki/values.yaml | 19 +++++ .../addons/logging/loki/deployed/chart.sls | 14 ++-- .../addons/logging/loki/deployed/init.sls | 36 ++++++++++ .../addons/logging/loki/deployed/macro.j2 | 61 ++++++++++++++++ .../logging/loki/deployed/pre-upgrade.sls | 48 +++++++++++++ .../loki/deployed/workaround-job-dep.sls | 30 ++++++++ salt/metalk8s/orchestrate/upgrade/pre.sls | 3 + 19 files changed, 294 insertions(+), 36 deletions(-) create mode 100644 salt/metalk8s/addons/logging/loki/deployed/macro.j2 create mode 100644 salt/metalk8s/addons/logging/loki/deployed/pre-upgrade.sls create mode 100644 salt/metalk8s/addons/logging/loki/deployed/workaround-job-dep.sls diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b2a9a01e4..36832fc465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,17 @@ (PR[#3910](https://github.com/scality/metalk8s/pull/3910)) - Bump Loki chart version to - [3.4.2](https://github.com/grafana/loki/releases/tag/helm-loki-3.4.2) + [3.4.3](https://github.com/grafana/loki/releases/tag/helm-loki-3.4.3) The Loki image has been bumped accordingly to [2.7.0](https://github.com/grafana/loki/releases/tag/v2.7.0) (PR[#3921](https://github.com/scality/metalk8s/pull/3921)) +### Bug fixes + +- Add a workaround for Loki chunk deletion waiting for a proper + fix in the next MetalK8s release + (PR[#3923](https://github.com/scality/metalk8s/pull/3923)) + ## Release 124.0.2 (in development) ### Bug fixes diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 6e1c3d0a65..b2e3cbc709 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -342,10 +342,13 @@ def task(self) -> types.TaskDict: Path("salt/metalk8s/addons/logging/loki/deployed/datasource.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/init.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/loki-configuration-secret.sls"), + Path("salt/metalk8s/addons/logging/loki/deployed/macro.j2"), Path("salt/metalk8s/addons/logging/loki/deployed/post-downgrade.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/post-upgrade.sls"), + Path("salt/metalk8s/addons/logging/loki/deployed/pre-upgrade.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/service.sls"), Path("salt/metalk8s/addons/logging/loki/deployed/service-configuration.sls"), + Path("salt/metalk8s/addons/logging/loki/deployed/workaround-job-dep.sls"), targets.TemplateFile( task_name="metalk8s-operator.sls", source=constants.ROOT.joinpath( diff --git a/charts/loki.yaml b/charts/loki.yaml index 8743ab5a8b..a414ec59ad 100644 --- a/charts/loki.yaml +++ b/charts/loki.yaml @@ -17,6 +17,8 @@ loki: checksum/config: '__slot__:salt:metalk8s_kubernetes.get_object_digest(kind="Secret", apiVersion="v1", namespace="metalk8s-logging", name="loki", path="data:config.yaml")' singleBinary: + targetModule: "all,table-manager" + replicas: '__var__(loki.spec.deployment.replicas)' persistence: diff --git a/charts/loki/Chart.yaml b/charts/loki/Chart.yaml index 5197b54f46..4e4c066dee 100644 --- a/charts/loki/Chart.yaml +++ b/charts/loki/Chart.yaml @@ -22,4 +22,4 @@ sources: - https://grafana.com/oss/loki/ - https://grafana.com/docs/loki/latest/ type: application -version: 3.4.2 +version: 3.4.3 diff --git a/charts/loki/README.md b/charts/loki/README.md index 896a3a75b5..d8f07175cb 100644 --- a/charts/loki/README.md +++ b/charts/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 3.4.2](https://img.shields.io/badge/Version-3.4.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.6.1](https://img.shields.io/badge/AppVersion-2.6.1-informational?style=flat-square) +![Version: 3.4.3](https://img.shields.io/badge/Version-3.4.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.6.1](https://img.shields.io/badge/AppVersion-2.6.1-informational?style=flat-square) Helm chart for Grafana Loki in simple, scalable mode diff --git a/charts/loki/docs/examples/README.md b/charts/loki/docs/examples/README.md index 327c6f103a..84cbae3e29 100644 --- a/charts/loki/docs/examples/README.md +++ b/charts/loki/docs/examples/README.md @@ -1,4 +1,4 @@ ## Introduction The Helm Charts found under the examples directory are getting started examples which you can use to deploy Loki using the Simple Scalable architecture quickly. Currently, the examples include: -- [Deploying Grafana Enterprise Logs (Loki in Enterprise mode)](https://github.com/grafana/helm-charts/tree/main/charts/loki-simple-scalable/docs/examples/enterprise) -- [Deploying Loki OSS](https://github.com/grafana/helm-charts/tree/main/charts/loki-simple-scalable/docs/examples/oss) +- [Deploying Grafana Enterprise Logs (Loki in Enterprise mode)](https://github.com/grafana/loki/tree/main/production/helm/loki/docs/examples/enterprise) +- [Deploying Loki OSS](https://github.com/grafana/loki/tree/main/production/helm/loki/docs/examples/oss) diff --git a/charts/loki/templates/_helpers.tpl b/charts/loki/templates/_helpers.tpl index 99b0a54112..7f268081ae 100644 --- a/charts/loki/templates/_helpers.tpl +++ b/charts/loki/templates/_helpers.tpl @@ -326,6 +326,75 @@ Return if ingress supports pathType. {{- or (eq (include "loki.ingress.isStable" .) "true") (and (eq (include "loki.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} {{- end -}} +{{/* +Generate list of ingress service paths based on deployment type +*/}} +{{- define "loki.ingress.servicePaths" -}} +{{- if (eq (include "loki.deployment.isScalable" .) "true") -}} +{{- include "loki.ingress.scalableServicePaths" . }} +{{- else -}} +{{- include "loki.ingress.singleBinaryServicePaths" . }} +{{- end -}} +{{- end -}} + +{{/* +Ingress service paths for scalable deployment +*/}} +{{- define "loki.ingress.scalableServicePaths" -}} +{{- include "loki.ingress.servicePath" (dict "ctx" . "svcName" "read" "paths" .Values.ingress.paths.read )}} +{{- include "loki.ingress.servicePath" (dict "ctx" . "svcName" "write" "paths" .Values.ingress.paths.write )}} +{{- end -}} + +{{/* +Ingress service paths for single binary deployment +*/}} +{{- define "loki.ingress.singleBinaryServicePaths" -}} +{{- include "loki.ingress.servicePath" (dict "ctx" . "svcName" "singleBinary" "paths" .Values.ingress.paths.singleBinary )}} +{{- end -}} + +{{/* +Ingress service path helper function +Params: + ctx = . context + svcName = service name without the "loki.fullname" part (ie. read, write) + paths = list of url paths to allow ingress for +*/}} +{{- define "loki.ingress.servicePath" -}} +{{- $ingressApiIsStable := eq (include "loki.ingress.isStable" .ctx) "true" -}} +{{- $ingressSupportsPathType := eq (include "loki.ingress.supportsPathType" .ctx) "true" -}} +{{- range .paths }} +- path: {{ . }} + {{- if $ingressSupportsPathType }} + pathType: Prefix + {{- end }} + backend: + {{- if $ingressApiIsStable }} + {{- $serviceName := include "loki.ingress.serviceName" (dict "ctx" $.ctx "svcName" $.svcName) }} + service: + name: {{ $serviceName }} + port: + number: 3100 + {{- else }} + serviceName: {{ $serviceName }} + servicePort: 3100 +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Ingress service name helper function +Params: + ctx = . context + svcName = service name without the "loki.fullname" part (ie. read, write) +*/}} +{{- define "loki.ingress.serviceName" -}} +{{- if (eq .svcName "singleBinary") }} +{{- printf "%s" (include "loki.fullname" .ctx) }} +{{- else }} +{{- printf "%s-%s" (include "loki.fullname" .ctx) .svcName }} +{{- end -}} +{{- end -}} + {{/* Create the service endpoint including port for MinIO. */}} diff --git a/charts/loki/templates/ingress.yaml b/charts/loki/templates/ingress.yaml index 4a2fddddd8..49e391191f 100644 --- a/charts/loki/templates/ingress.yaml +++ b/charts/loki/templates/ingress.yaml @@ -1,7 +1,5 @@ {{- if .Values.ingress.enabled }} -{{- $ingressApiIsStable := eq (include "loki.ingress.isStable" .) "true" -}} {{- $ingressSupportsIngressClassName := eq (include "loki.ingress.supportsIngressClassName" .) "true" -}} -{{- $ingressSupportsPathType := eq (include "loki.ingress.supportsPathType" .) "true" -}} apiVersion: {{ include "loki.ingress.apiVersion" . }} kind: Ingress metadata: @@ -33,23 +31,6 @@ spec: - host: {{ . | quote }} http: paths: - {{- range $svcName, $paths := $.Values.ingress.paths }} - {{- range $paths }} - - path: {{ . }} - {{- if $ingressSupportsPathType }} - pathType: Prefix - {{- end }} - backend: - {{- if $ingressApiIsStable }} - service: - name: {{ include "loki.fullname" $ }}-{{ $svcName }} - port: - number: 3100 - {{- else }} - serviceName: {{ include "loki.fullname" $ }}-{{ $svcName }} - servicePort: 3100 - {{- end }} - {{- end }} - {{- end }} + {{- include "loki.ingress.servicePaths" $ | indent 10}} {{- end }} {{- end }} diff --git a/charts/loki/templates/read/statefulset-read.yaml b/charts/loki/templates/read/statefulset-read.yaml index 06bca619f5..e777469d67 100644 --- a/charts/loki/templates/read/statefulset-read.yaml +++ b/charts/loki/templates/read/statefulset-read.yaml @@ -61,7 +61,7 @@ spec: imagePullPolicy: {{ .Values.loki.image.pullPolicy }} args: - -config.file=/etc/loki/config/config.yaml - - -target=read + - -target={{ .Values.read.targetModule }} {{- with .Values.read.extraArgs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/loki/templates/single-binary/statefulset.yaml b/charts/loki/templates/single-binary/statefulset.yaml index 1c4dbb3017..30dccc6776 100644 --- a/charts/loki/templates/single-binary/statefulset.yaml +++ b/charts/loki/templates/single-binary/statefulset.yaml @@ -52,7 +52,7 @@ spec: imagePullPolicy: {{ .Values.loki.image.pullPolicy }} args: - -config.file=/etc/loki/config/config.yaml - - -target=all + - -target={{ .Values.singleBinary.targetModule }} {{- with .Values.singleBinary.extraArgs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/loki/templates/tokengen/job-tokengen.yaml b/charts/loki/templates/tokengen/job-tokengen.yaml index ec7cffddc4..489e550003 100644 --- a/charts/loki/templates/tokengen/job-tokengen.yaml +++ b/charts/loki/templates/tokengen/job-tokengen.yaml @@ -46,7 +46,7 @@ spec: imagePullPolicy: {{ .Values.loki.image.pullPolicy }} args: - -config.file=/etc/loki/config/config.yaml - - -target=tokengen + - -target={{ .Values.enterprise.tokengen.targetModule }} - -tokengen.token-file=/shared/admin-token {{- with .Values.enterprise.tokengen.extraArgs }} {{- toYaml . | nindent 12 }} diff --git a/charts/loki/templates/write/statefulset-write.yaml b/charts/loki/templates/write/statefulset-write.yaml index 82c8a374e0..d09a1dfb08 100644 --- a/charts/loki/templates/write/statefulset-write.yaml +++ b/charts/loki/templates/write/statefulset-write.yaml @@ -53,7 +53,7 @@ spec: imagePullPolicy: {{ .Values.loki.image.pullPolicy }} args: - -config.file=/etc/loki/config/config.yaml - - -target=write + - -target={{ .Values.write.targetModule }} {{- with .Values.write.extraArgs }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/loki/values.yaml b/charts/loki/values.yaml index 45e01be603..0af1434b6c 100644 --- a/charts/loki/values.yaml +++ b/charts/loki/values.yaml @@ -335,6 +335,8 @@ enterprise: tokengen: # -- Whether the job should be part of the deployment enabled: true + # -- Comma-separated list of Loki modules to load for tokengen + targetModule: "tokengen" # -- Additional CLI arguments for the `tokengen` target extraArgs: [] # -- Additional Kubernetes environment @@ -747,6 +749,8 @@ write: selectorLabels: {} # -- Labels for ingestor service serviceLabels: {} + # -- Comma-separated list of Loki modules to load for the write + targetModule: "write" # -- Additional CLI args for the write extraArgs: [] # -- Environment variables to add to the write pods @@ -818,6 +822,8 @@ read: selectorLabels: {} # -- Labels for read service serviceLabels: {} + # -- Comma-separated list of Loki modules to load for the read + targetModule: "read" # -- Additional CLI args for the read extraArgs: [] # -- Environment variables to add to the read pods @@ -885,6 +891,8 @@ singleBinary: podAnnotations: {} # -- Additional selecto labels for each `single binary` pod selectorLabels: {} + # -- Comma-separated list of Loki modules to load for the single binary + targetModule: "all" # -- Labels for single binary service extraArgs: [] # -- Environment variables to add to the single binary pods @@ -948,6 +956,17 @@ ingress: - /loki/api/v1/rules - /prometheus/api/v1/rules - /prometheus/api/v1/alerts + singleBinary: + - /api/prom/push + - /loki/api/v1/push + - /api/prom/tail + - /loki/api/v1/tail + - /loki/api + - /api/prom/rules + - /loki/api/v1/rules + - /prometheus/api/v1/rules + - /prometheus/api/v1/alerts + hosts: - loki.example.com # tls: diff --git a/salt/metalk8s/addons/logging/loki/deployed/chart.sls b/salt/metalk8s/addons/logging/loki/deployed/chart.sls index 51c74bc5bf..ff0c8d72ba 100644 --- a/salt/metalk8s/addons/logging/loki/deployed/chart.sls +++ b/salt/metalk8s/addons/logging/loki/deployed/chart.sls @@ -16,7 +16,7 @@ metadata: app.kubernetes.io/name: loki app.kubernetes.io/part-of: metalk8s app.kubernetes.io/version: 2.6.1 - helm.sh/chart: loki-3.4.2 + helm.sh/chart: loki-3.4.3 heritage: metalk8s name: loki namespace: metalk8s-logging @@ -30,7 +30,7 @@ metadata: app.kubernetes.io/name: loki app.kubernetes.io/part-of: metalk8s app.kubernetes.io/version: 2.6.1 - helm.sh/chart: loki-3.4.2 + helm.sh/chart: loki-3.4.3 heritage: metalk8s name: loki-memberlist namespace: metalk8s-logging @@ -56,7 +56,7 @@ metadata: app.kubernetes.io/name: loki app.kubernetes.io/part-of: metalk8s app.kubernetes.io/version: 2.6.1 - helm.sh/chart: loki-3.4.2 + helm.sh/chart: loki-3.4.3 heritage: metalk8s prometheus.io/service-monitor: 'false' variant: headless @@ -82,7 +82,7 @@ metadata: app.kubernetes.io/name: loki app.kubernetes.io/part-of: metalk8s app.kubernetes.io/version: 2.6.1 - helm.sh/chart: loki-3.4.2 + helm.sh/chart: loki-3.4.3 heritage: metalk8s name: loki namespace: metalk8s-logging @@ -112,7 +112,7 @@ metadata: app.kubernetes.io/name: loki app.kubernetes.io/part-of: metalk8s app.kubernetes.io/version: 2.6.1 - helm.sh/chart: loki-3.4.2 + helm.sh/chart: loki-3.4.3 heritage: metalk8s name: loki namespace: metalk8s-logging @@ -150,7 +150,7 @@ spec: containers: - args: - -config.file=/etc/loki/config/config.yaml - - -target=all + - -target=all,table-manager image: {% endraw -%}{{ build_image_name("loki", False) }}{%- raw %}:2.7.0 imagePullPolicy: IfNotPresent name: single-binary @@ -230,7 +230,7 @@ metadata: app.kubernetes.io/name: loki app.kubernetes.io/part-of: metalk8s app.kubernetes.io/version: 2.6.1 - helm.sh/chart: loki-3.4.2 + helm.sh/chart: loki-3.4.3 heritage: metalk8s metalk8s.scality.com/monitor: '' name: loki diff --git a/salt/metalk8s/addons/logging/loki/deployed/init.sls b/salt/metalk8s/addons/logging/loki/deployed/init.sls index b162ffaaec..139e4a6011 100644 --- a/salt/metalk8s/addons/logging/loki/deployed/init.sls +++ b/salt/metalk8s/addons/logging/loki/deployed/init.sls @@ -5,6 +5,7 @@ include: - .chart - .service - .datasource + - .workaround-job-dep {#- Due to a change of Loki StatefulSet labelSelector in 124.1.0, which is immutable field Manually delete the Loki StatefulSet object if it's an old one @@ -33,3 +34,38 @@ Delete the old Loki StatefulSet: - sls: metalk8s.addons.logging.loki.deployed.chart {%- endif %} + +{#- Delete the Loki pre-upgrade Job since we will deploy the CronJob #} +Ensure Loki pre-upgrade Workaround Job is deleted: + metalk8s_kubernetes.object_absent: + - apiVersion: batch/v1 + - kind: Job + - name: loki-cleaner-wa-pre-upgrade + - namespace: metalk8s-logging + - wait: + attempts: 30 + sleep: 10 + - require: + - sls: metalk8s.addons.logging.loki.deployed.chart + +{#- Due to a bug in Loki that prevent deletion of the old chunks, + Add a CronJob to delete old chunks waiting for a proper fix #} +{%- from "metalk8s/addons/logging/loki/deployed/macro.j2" import workaround_job_spec with context %} + +Create Loki Cleaner Workaround CronJob: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: batch/v1 + kind: CronJob + metadata: + name: loki-cleaner-wa + namespace: metalk8s-logging + spec: + schedule: "@daily" + concurrencyPolicy: Replace + jobTemplate: + spec: + {{ workaround_job_spec("/var/loki") | indent(14) }} + - require: + - sls: metalk8s.addons.logging.loki.deployed.workaround-job-dep + - sls: metalk8s.addons.logging.loki.deployed.chart diff --git a/salt/metalk8s/addons/logging/loki/deployed/macro.j2 b/salt/metalk8s/addons/logging/loki/deployed/macro.j2 new file mode 100644 index 0000000000..ba0f393a92 --- /dev/null +++ b/salt/metalk8s/addons/logging/loki/deployed/macro.j2 @@ -0,0 +1,61 @@ +{%- macro workaround_job_spec(data_dir) -%} +{%- from "metalk8s/repo/macro.sls" import build_image_name with context %} +{%- set loki_defaults = salt.slsutil.renderer('salt://metalk8s/addons/logging/loki/config/loki.yaml', saltenv=saltenv) %} +{%- set loki = salt.metalk8s_service_configuration.get_service_conf('metalk8s-logging', 'metalk8s-loki-config', loki_defaults) %} + +{%- set retention_period = loki.spec.config.table_manager.retention_period %} +{#- NOTE: Consider the last index using `boltdb` is the one currently used #} +{%- set index_period = (loki.spec.config.schema_config.configs | selectattr("store", "==", "boltdb") | list | last).index.period %} + +{#- Let's consider we only support period in hours for the moment #} +{%- if retention_period[-1] != "h" %} + {{ raise("Invalid Loki table manager retention_period, should be in hours") }} +{%- elif index_period[-1] != "h" %} + {{ raise("Invalid Loki index period, should be in hours") }} +{%- else %} + {#- Let's remove file older than retention time + index time + 3 days #} + {%- set max_age = (( + (retention_period[:-1] | int) / 24 + + (index_period[:-1] | int) / 24 + )| int) + 3 %} +{%- endif %} + +template: + spec: + restartPolicy: OnFailure + serviceAccountName: loki-cleaner-wa + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/bootstrap + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/infra + operator: Exists + containers: + - name: loki-cleaner-wa-container + image: {{ build_image_name("metalk8s-utils") }} + env: + - name: MAX_AGE + value: "{{ max_age }}" + - name: LOKI_DIR + value: "{{ data_dir }}" + command: + - bash + - "-c" + args: + - |- + set -e + for i in $(seq 0 $(("$(kubectl get sts loki -o jsonpath='{.spec.replicas}')" - 1))); do + if ! kubectl wait pod loki-$i --for condition=Ready --timeout=0; then + echo "Pod loki-$i is not running restart it" + kubectl delete pod loki-$i + echo "Waiting for loki-$i to restart" + kubectl wait pod loki-$i --for condition=Ready --timeout=300s + echo "Pod loki-$i is now running" + fi + echo "Cleaning loki-$i chunks older than $MAX_AGE days" + kubectl exec -t loki-$i -- find "$LOKI_DIR/chunks/" -type f -mtime +"$MAX_AGE" ! -name '*.json' -delete + echo "Cleaning of loki-$i done" + done + +{%- endmacro -%} diff --git a/salt/metalk8s/addons/logging/loki/deployed/pre-upgrade.sls b/salt/metalk8s/addons/logging/loki/deployed/pre-upgrade.sls new file mode 100644 index 0000000000..a83043d30a --- /dev/null +++ b/salt/metalk8s/addons/logging/loki/deployed/pre-upgrade.sls @@ -0,0 +1,48 @@ +{#- Due to a bug in Loki that prevent deletion of the old chunks, + Some Loki pods might be stuck in CrashLoopBackOff because the volume is full + Run a Job to delete old chunks before the upgrade to restore + NOTE: This is automatically handled since 124.1.0 + So this Logic can be removed in `development/126.0` #} + +{%- set loki_sts = salt.metalk8s_kubernetes.get_object( + kind="StatefulSet", + apiVersion="apps/v1", + name="loki", + namespace="metalk8s-logging" +) %} +{%- set loki_wa_job = salt.metalk8s_kubernetes.get_object( + kind="Job", + apiVersion="batch/v1", + name="loki-cleaner-wa-pre-upgrade", + namespace="metalk8s-logging" +) %} + +{%- if not loki_wa_job and loki_sts and salt.pkg.version_cmp( + loki_sts["metadata"]["labels"]["metalk8s.scality.com/version"], + "124.1.0" + ) == -1 %} + +include: + - .workaround-job-dep + + {%- from "metalk8s/addons/logging/loki/deployed/macro.j2" import workaround_job_spec with context %} + +Create Loki Cleaner Workaround Job: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: batch/v1 + kind: Job + metadata: + name: loki-cleaner-wa-pre-upgrade + namespace: metalk8s-logging + spec: + {{ workaround_job_spec("/data/loki") | indent(10) }} + - require: + - sls: metalk8s.addons.logging.loki.deployed.workaround-job-dep + +{%- else %} + +Nothing to do before upgrading: + test.nop: [] + +{%- endif %} diff --git a/salt/metalk8s/addons/logging/loki/deployed/workaround-job-dep.sls b/salt/metalk8s/addons/logging/loki/deployed/workaround-job-dep.sls new file mode 100644 index 0000000000..0b9c6613ec --- /dev/null +++ b/salt/metalk8s/addons/logging/loki/deployed/workaround-job-dep.sls @@ -0,0 +1,30 @@ +{#- Due to a bug in Loki that prevent deletion of the old chunks, + Use a Job to delete old chunks waiting for a proper fix + This state deploy the Service Account and ClusterRoleBinding used by the Workload Job #} + +Create Loki Cleaner Workaround Service Account: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: loki-cleaner-wa + namespace: metalk8s-logging + +Create Loki Cleaner Workaround ClusterRoleBinding: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: loki-cleaner-wa + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: loki-cleaner-wa + namespace: metalk8s-logging + - require: + - metalk8s_kubernetes: Create Loki Cleaner Workaround Service Account diff --git a/salt/metalk8s/orchestrate/upgrade/pre.sls b/salt/metalk8s/orchestrate/upgrade/pre.sls index 8ae0b2f78e..427fb35b55 100644 --- a/salt/metalk8s/orchestrate/upgrade/pre.sls +++ b/salt/metalk8s/orchestrate/upgrade/pre.sls @@ -2,6 +2,9 @@ # NOTE: This state should be called by salt-master using the saltenv of # the destination version (salt-master should have been upgraded) +include: + - metalk8s.addons.logging.loki.deployed.pre-upgrade + {%- set cp_nodes = salt.metalk8s.minions_by_role('master') %} {%- if cp_nodes|length == 1 %}