upgrade fluent-bit

Makefile

@@ -50,6 +50,12 @@ docker_build:
 
 docker_pull_test_images:
 	@eval $$(minikube docker-env); \
+	    docker pull postgres:12.11; \
+	    docker pull redis:4; \
+	    docker pull curlimages/curl:8.4.0; \
+	    docker pull fluent/fluent-bit:2.1.10; \
+	    docker pull minio/minio:RELEASE.2023-10-14T05-17-22Z; \
+	    docker pull lachlanevenson/k8s-kubectl:v1.10.3; \
 	    docker pull $(PLATFORMAPI_IMAGE); \
 	    docker pull $(PLATFORMADMIN_IMAGE); \
 	    docker pull $(PLATFORMAUTHAPI_IMAGE); \

charts/platform-monitoring/.gitignore

@@ -0,0 +1 @@
+charts/

charts/platform-monitoring/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: k8s-resources
+  repository: https://neuro-inc.github.io/helm-charts
+  version: 1.0.1
+digest: sha256:b7235176cb0cd47c2c5ac86a6fbec6cb268fe777e40a905772d8549bb04d9715
+generated: "2023-10-13T17:01:35.552161764+03:00"

charts/platform-monitoring/files/fluent-bit/fluent-bit.conf

@@ -1,12 +1,12 @@
 [SERVICE]
     Flush        1
     Daemon       Off
-    Log_Level    info
+    Log_Level    {{ .Values.fluentbit.logLevel }}
     Parsers_File parsers.conf
 
 [INPUT]
     Name             tail
-    Path             /var/log/containers/*.log
+    Path             /var/log/containers/job-*_{{ .Values.jobsNamespace }}_*.log
 {{- if eq .Values.containerRuntime.name "docker" }}
     Parser           docker
 {{- else }}
@@ -40,16 +40,38 @@
     Retry_Limit     False
     Type            fluent-bit
     Time_Key        @timestamp
+    Generate_ID     On
     Replace_Dots    On
     Logstash_Format On
     Logstash_Prefix logstash
 {{- else }}
 
 [OUTPUT]
-    Name        forward
-    Match       kube.var.log.containers.job-*
-    Host        fluentd-service
-    Port        {{ .Values.fluentd.forwardPort }}
-    tls         off
-    Retry_Limit False
+    Name             s3
+    Match            kube.var.log.containers.job-*
+{{- if eq .Values.logs.persistence.type "aws" }}
+{{- if .Values.logs.persistence.aws.endpoint }}
+    endpoint         {{ .Values.logs.persistence.aws.endpoint }}
+{{- end }}
+    region           {{ .Values.logs.persistence.aws.region }}
+    bucket           {{ .Values.logs.persistence.aws.bucket }}
+{{- else if eq .Values.logs.persistence.type "minio" }}
+    endpoint         {{ .Values.logs.persistence.minio.url }}
+    region           {{ .Values.logs.persistence.minio.region }}
+    bucket           {{ .Values.logs.persistence.minio.bucket }}
+{{- else if eq .Values.logs.persistence.type "gcp" }}
+    endpoint         {{ include "platformMonitoring.minioGateway.endpoint" . }}
+    region           {{ .Values.logs.persistence.gcp.location }}
+    bucket           {{ .Values.logs.persistence.gcp.bucket }}
+{{- else if eq .Values.logs.persistence.type "azure" }}
+    endpoint         {{ include "platformMonitoring.minioGateway.endpoint" . }}
+    region           minio
+    bucket           {{ .Values.logs.persistence.azure.bucket }}
+{{- end }}
+    total_file_size  1M
+    upload_timeout   1m
+    use_put_object   On
+    s3_key_format    /$TAG/%Y%m%d%H%M__$INDEX.gz
+    static_file_path On
+    json_date_key    time
 {{- end }}

charts/platform-monitoring/templates/_helpers.tpl

@@ -28,11 +28,11 @@
 {{- end -}}
 {{- end -}}
 
-{{- define "platformMonitoring.fluentd.fullname" -}}
-{{- if .Values.fluentd.fullnameOverride -}}
-{{- .Values.fluentd.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- define "platformMonitoring.minioGateway.fullname" -}}
+{{- if .Values.minioGateway.fullnameOverride -}}
+{{- .Values.minioGateway.fullnameOverride | trunc 63 | trimSuffix "-" -}}
 {{- else -}}
-{{- $name := default "fluent-bit" .Values.fluentd.nameOverride -}}
+{{- $name := default "minio-gateway" .Values.minioGateway.nameOverride -}}
 {{- if contains $name .Release.Name -}}
 {{- .Release.Name | trunc 63 | trimSuffix "-" -}}
 {{- else -}}
@@ -41,6 +41,11 @@
 {{- end -}}
 {{- end -}}
 
+{{- define "platformMonitoring.minioGateway.endpoint" -}}
+{{- $serviceName := include "platformMonitoring.minioGateway.fullname" . -}}
+{{- printf "http://%s:%s" $serviceName (toString .Values.minioGateway.port) -}}
+{{- end -}}
+
 {{- define "platformMonitoring.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" -}}
 {{- end -}}
@@ -52,40 +57,6 @@ heritage: {{ .Release.Service | quote }}
 release: {{ .Release.Name | quote }}
 {{- end -}}
 
-{{- define "platformMonitoring.minio.probes" -}}
-livenessProbe:
-  httpGet:
-    path: /minio/health/live
-    port: {{ .Values.minio.port }}
-{{- /* if dict is empty 'with' block is not executed, creating default dummy dict */ -}}
-{{- with (.Values.minio.livenessProbe | default (dict "dummy" "dummy")) }}
-  initialDelaySeconds: {{ .initialDelaySeconds | default 5 }}
-  periodSeconds: {{ .periodSeconds | default 10 }}
-  timeoutSeconds: {{ .timeoutSeconds | default 1 }}
-  successThreshold: {{ .successThreshold | default 1 }}
-  failureThreshold: {{ .failureThreshold | default 3 }}
-{{- end }}
-readinessProbe:
-  httpGet:
-    path: /minio/health/ready
-    port: {{ .Values.minio.port }}
-{{- /* if dict is empty 'with' block is not executed, creating default dummy dict */ -}}
-{{- with (.Values.minio.readinessProbe | default (dict "dummy" "dummy")) }}
-  initialDelaySeconds: {{ .initialDelaySeconds | default 5 }}
-  periodSeconds: {{ .periodSeconds | default 10 }}
-  timeoutSeconds: {{ .timeoutSeconds | default 1 }}
-  successThreshold: {{ .successThreshold | default 1 }}
-  failureThreshold: {{ .failureThreshold | default 3 }}
-{{- end }}
-{{- end -}}
-
-{{- define "platformMonitoring.minio.resources" -}}
-{{- if .Values.minio.resources }}
-resources:
-{{ toYaml .Values.minio.resources | indent 2 }}
-{{- end }}
-{{- end -}}
-
 {{- define "platformMonitoring.logs.storage.s3.keyPrefixFormat" -}}
 kube.var.log.containers.{pod_name}_{namespace_name}_{container_name}
 {{- end -}}

charts/platform-monitoring/templates/deployment.yml

@@ -150,9 +150,7 @@ spec:
           value: {{ .Values.logs.persistence.aws.bucket | quote }}
         - name: NP_MONITORING_S3_JOB_LOGS_KEY_PREFIX_FORMAT
           value: {{ include "platformMonitoring.logs.storage.s3.keyPrefixFormat" . | quote }}
-        {{- end }}
-
-      {{- if eq .Values.logs.persistence.type "gcp" }}
+        {{- else if eq .Values.logs.persistence.type "gcp" }}
         - name: NP_MONITORING_LOGS_STORAGE_TYPE
           value: s3
         - name: NP_MONITORING_S3_REGION
@@ -162,14 +160,12 @@ spec:
         - name: NP_MONITORING_S3_SECRET_ACCESS_KEY
           value: minio_secret_key
         - name: NP_MONITORING_S3_ENDPOINT_URL
-          value: "http://127.0.0.1:{{ .Values.minio.port }}"
+          value: {{ include "platformMonitoring.minioGateway.endpoint" . }}
         - name: NP_MONITORING_S3_JOB_LOGS_BUCKET_NAME
           value: {{ .Values.logs.persistence.gcp.bucket | quote }}
         - name: NP_MONITORING_S3_JOB_LOGS_KEY_PREFIX_FORMAT
           value: {{ include "platformMonitoring.logs.storage.s3.keyPrefixFormat" . | quote }}
-      {{- end }}
-
-      {{- if eq .Values.logs.persistence.type "azure" }}
+        {{- else if eq .Values.logs.persistence.type "azure" }}
         - name: NP_MONITORING_LOGS_STORAGE_TYPE
           value: s3
         - name: NP_MONITORING_S3_REGION
@@ -185,14 +181,12 @@ spec:
               name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
               key: account_key
         - name: NP_MONITORING_S3_ENDPOINT_URL
-          value: "http://127.0.0.1:{{ .Values.minio.port }}"
+          value: {{ include "platformMonitoring.minioGateway.endpoint" . }}
         - name: NP_MONITORING_S3_JOB_LOGS_BUCKET_NAME
           value: {{ .Values.logs.persistence.azure.bucket | quote }}
         - name: NP_MONITORING_S3_JOB_LOGS_KEY_PREFIX_FORMAT
           value: {{ include "platformMonitoring.logs.storage.s3.keyPrefixFormat" . | quote }}
-      {{- end }}
-
-        {{- if eq .Values.logs.persistence.type "minio" }}
+        {{- else if eq .Values.logs.persistence.type "minio" }}
         - name: NP_MONITORING_LOGS_STORAGE_TYPE
           value: s3
         - name: NP_MONITORING_S3_ENDPOINT_URL
@@ -213,9 +207,7 @@ spec:
           value: {{ .Values.logs.persistence.minio.bucket | quote }}
         - name: NP_MONITORING_S3_JOB_LOGS_KEY_PREFIX_FORMAT
           value: {{ include "platformMonitoring.logs.storage.s3.keyPrefixFormat" . | quote }}
-        {{- end }}
-
-        {{- if eq .Values.logs.persistence.type "es" }}
+        {{- else if eq .Values.logs.persistence.type "es" }}
         - name: NP_MONITORING_LOGS_STORAGE_TYPE
           value: elasticsearch
         - name: NP_MONITORING_ES_HOSTS
@@ -227,50 +219,6 @@ spec:
         - mountPath: {{ include "platformMonitoring.kubeAuthMountRoot" . }}
           name: kube-api-data
           readOnly: true
-
-      {{- if eq .Values.logs.persistence.type "gcp" }}
-      - name: minio
-        image: {{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}
-        args:
-        - gateway
-        - gcs
-        - {{ .Values.logs.persistence.gcp.project }}
-        env:
-        - name: MINIO_ROOT_USER
-          value: minio_access_key
-        - name: MINIO_ROOT_PASSWORD
-          value: minio_secret_key
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /accounts/key_json
-        volumeMounts:
-        - name: gcs-credentials
-          mountPath: /accounts
-          readOnly: true
-{{ include "platformMonitoring.minio.resources" . | indent 8 }}
-{{ include "platformMonitoring.minio.probes" . | indent 8 }}
-      {{- end }}
-
-      {{- if eq .Values.logs.persistence.type "azure" }}
-      - name: minio
-        image: {{ .Values.minio.image.repository }}:{{ .Values.minio.image.tag }}
-        args:
-        - gateway
-        - azure
-        env:
-        - name: MINIO_ROOT_USER
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: account_name
-        - name: MINIO_ROOT_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-              key: account_key
-{{ include "platformMonitoring.minio.resources" . | indent 8 }}
-{{ include "platformMonitoring.minio.probes" . | indent 8 }}
-      {{- end }}
-
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 6 }}
       {{- end }}
@@ -289,8 +237,3 @@ spec:
               items:
               - key: ca.crt
                 path: ca.crt
-      {{- if eq .Values.logs.persistence.type "gcp" }}
-      - name: gcs-credentials
-        secret:
-          secretName: {{ include "platformMonitoring.logs.storage.keySecret" . }}
-      {{- end }}