Merge pull request #95 from owncloud/healthcheck

add healthchecks and prepare monitoring
owncloud · Nov 9, 2022 · 0d9672c · 0d9672c
2 parents 434c507 + fec7751
commit 0d9672c
Show file tree

Hide file tree

Showing 56 changed files with 674 additions and 66 deletions.
diff --git a/charts/ocis/docs/values-desc-table.adoc b/charts/ocis/docs/values-desc-table.adoc
@@ -30,6 +30,12 @@ a| [subs=-attributes]
 a| [subs=-attributes]
 `3`
 | Sets minimum replicas for autoscaling.
+| debug.profiling
+a| [subs=-attributes]
++bool+
+a| [subs=-attributes]
+`false`
+| Profiling enables the http://<pod>:<debug-metrics>/debug/pprof endpoint to inspect various Go runtime internals. You can use the endpoint on your machine by forwarding the port, eg: `kubectl port-forward -n ocis pod/auth-basic-8587dc9d64-fs24l 9147:9147` and then accessing the port on https://localhost:9147/debug/pprof or using the pprof command line tool: `go tool pprof -web http://localhost:9147/debug/pprof/symbol\?seconds\=10`
 | deploymentStrategy
 a| [subs=-attributes]
 +object+

diff --git a/charts/ocis/docs/values.adoc.yaml b/charts/ocis/docs/values.adoc.yaml
@@ -23,6 +23,13 @@ logging:
   # Not recommended for production installations.
   color: "false"
 
+# Debug settings for the oCIS service
+debug:
+  # -- Profiling enables the http://<pod>:<debug-metrics>/debug/pprof endpoint to inspect various Go runtime internals.
+  # You can use the endpoint on your machine by forwarding the port, eg: `kubectl port-forward -n ocis pod/auth-basic-8587dc9d64-fs24l 9147:9147`
+  # and then accessing the port on https://localhost:9147/debug/pprof or using the pprof command line tool: `go tool pprof -web http://localhost:9147/debug/pprof/symbol\?seconds\=10`
+  profiling: false
+
 # -- Deployment strategy.
 deploymentStrategy:
   type: RollingUpdate

diff --git a/charts/ocis/templates/app-provider/deployment.yaml b/charts/ocis/templates/app-provider/deployment.yaml
@@ -48,8 +48,13 @@ spec:
             - name: APP_PROVIDER_LOG_PRETTY
               value: "{{ $.Values.logging.pretty }}"
 
+            - name: APP_PROVIDER_DEBUG_PPROF
+              value: "{{ $.Values.debug.profiling }}"
+
             - name: APP_PROVIDER_GRPC_ADDR
               value: 0.0.0.0:9164
+            - name: APP_PROVIDER_DEBUG_ADDR
+              value: 0.0.0.0:9165
 
             - name: APP_PROVIDER_EXTERNAL_ADDR
               value: app-provider-{{ regexReplaceAll "\\W+" (lower $officeSuite.name) "_" }}:9164
@@ -76,10 +81,23 @@ spec:
                   name: {{ $.Values.secretRefs.jwtSecretRef }}
                   key: jwt-secret
 
+          livenessProbe:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - curl --silent --fail http://frontend:9140/app/list | grep {{ $officeSuite.name }}
+            timeoutSeconds: 10
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            failureThreshold: 3
+
           resources: {{ toYaml $.Values.resources | nindent 12 }}
           ports:
             - name: grpc
               containerPort: 9164
+            - name: metrics-debug
+              containerPort: 9165
           volumeMounts:
             - name: tmp-volume
               mountPath: /tmp

diff --git a/charts/ocis/templates/app-provider/service.yaml b/charts/ocis/templates/app-provider/service.yaml
@@ -5,17 +5,22 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name:  app-provider-{{ regexReplaceAll "\\W+" (lower $officeSuite.name) "_" }}
+  name: app-provider-{{ regexReplaceAll "\\W+" (lower $officeSuite.name) "_" }}
   namespace: {{ template "ocis.namespace" $ }}
   labels:
+    app: app-provider-{{ regexReplaceAll "\\W+" (lower $officeSuite.name) "_" }}
+    ocis-metrics: enabled
     {{- include "ocis.labels" $ | nindent 4 }}
 spec:
+  selector:
+    app:  app-provider-{{ regexReplaceAll "\\W+" (lower $officeSuite.name) "_" }}
   ports:
     - name: grpc
       port: 9164
       protocol: TCP
-  selector:
-    app:  app-provider-{{ regexReplaceAll "\\W+" (lower $officeSuite.name) "_" }}
+    - name: metrics-debug
+      port: 9165
+      protocol: TCP
 {{ end }}
 {{ end }}
 {{ end }}
diff --git a/charts/ocis/templates/app-registry/deployment.yaml b/charts/ocis/templates/app-registry/deployment.yaml
@@ -47,8 +47,13 @@ spec:
             - name: APP_REGISTRY_LOG_PRETTY
               value: "{{ .Values.logging.pretty }}"
 
+            - name: APP_REGISTRY_DEBUG_PPROF
+              value: "{{ .Values.debug.profiling }}"
+
             - name: APP_REGISTRY_GRPC_ADDR
               value: 0.0.0.0:9242
+            - name: APP_REGISTRY_DEBUG_ADDR
+              value: 0.0.0.0:9243
 
             - name: REVA_GATEWAY
               value: gateway:9142
@@ -59,10 +64,21 @@ spec:
                   name: {{ .Values.secretRefs.jwtSecretRef }}
                   key: jwt-secret
 
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics-debug
+            timeoutSeconds: 10
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            failureThreshold: 3
+
           resources: {{ toYaml .Values.resources | nindent 12 }}
           ports:
             - name: http
               containerPort: 9242
+            - name: metrics-debug
+              containerPort: 9243
           volumeMounts:
             - name: tmp-volume
               mountPath: /tmp

diff --git a/charts/ocis/templates/app-registry/service.yaml b/charts/ocis/templates/app-registry/service.yaml
@@ -5,12 +5,17 @@ metadata:
   name: app-registry
   namespace: {{ template "ocis.namespace" . }}
   labels:
+    app: app-registry
+    ocis-metrics: enabled
     {{- include "ocis.labels" . | nindent 4 }}
 spec:
+  selector:
+    app: app-registry
   ports:
     - name: grpc
       port: 9242
       protocol: TCP
-  selector:
-    app: app-registry
+    - name: metrics-debug
+      port: 9243
+      protocol: TCP
 {{ end }}
diff --git a/charts/ocis/templates/auth-basic/deployment.yaml b/charts/ocis/templates/auth-basic/deployment.yaml
@@ -47,8 +47,13 @@ spec:
             - name: AUTH_BASIC_LOG_PRETTY
               value: "{{ .Values.logging.pretty }}"
 
+            - name: AUTH_BASIC_DEBUG_PPROF
+              value: "{{ .Values.debug.profiling }}"
+
             - name: AUTH_BASIC_GRPC_ADDR
               value: 0.0.0.0:9146
+            - name: AUTH_BASIC_DEBUG_ADDR
+              value: 0.0.0.0:9147
 
             - name: REVA_GATEWAY
               value: gateway:9142
@@ -147,10 +152,21 @@ spec:
                   name: {{ .Values.secretRefs.jwtSecretRef }}
                   key: jwt-secret
 
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics-debug
+            timeoutSeconds: 10
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            failureThreshold: 3
+
           resources: {{ toYaml .Values.resources | nindent 12 }}
           ports:
-          - name: grpc
-            containerPort: 9146
+            - name: grpc
+              containerPort: 9146
+            - name: metrics-debug
+              containerPort: 9147
           volumeMounts:
             - name: ocis-config-tmp
               mountPath: /etc/ocis # we mount that volume only to apply fsGroup to that path

diff --git a/charts/ocis/templates/auth-basic/service.yaml b/charts/ocis/templates/auth-basic/service.yaml
@@ -5,12 +5,17 @@ metadata:
   name: auth-basic
   namespace: {{ template "ocis.namespace" . }}
   labels:
+    app: auth-basic
+    ocis-metrics: enabled
     {{- include "ocis.labels" . | nindent 4 }}
 spec:
+  selector:
+    app: auth-basic
   ports:
     - name: grpc
       port: 9146
       protocol: TCP
-  selector:
-    app: auth-basic
+    - name: metrics-debug
+      port: 9147
+      protocol: TCP
 {{ end }}
diff --git a/charts/ocis/templates/auth-machine/deployment.yaml b/charts/ocis/templates/auth-machine/deployment.yaml
@@ -46,8 +46,13 @@ spec:
             - name: AUTH_MACHINE_LOG_PRETTY
               value: "{{ .Values.logging.pretty }}"
 
+            - name: AUTH_MACHINE_DEBUG_PPROF
+              value: "{{ .Values.debug.profiling }}"
+
             - name: AUTH_MACHINE_GRPC_ADDR
               value: 0.0.0.0:9166
+            - name: AUTH_MACHINE_DEBUG_ADDR
+              value: 0.0.0.0:9167
 
             - name: REVA_GATEWAY
               value: gateway:9142
@@ -64,10 +69,21 @@ spec:
                   name: {{ .Values.secretRefs.machineAuthApiKeySecretRef }}
                   key: machine-auth-api-key
 
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics-debug
+            timeoutSeconds: 10
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            failureThreshold: 3
+
           resources: {{ toYaml .Values.resources | nindent 12 }}
           ports:
             - name: grpc
               containerPort: 9166
+            - name: metrics-debug
+              containerPort: 9167
           volumeMounts:
             - name: tmp-volume
               mountPath: /tmp

diff --git a/charts/ocis/templates/auth-machine/service.yaml b/charts/ocis/templates/auth-machine/service.yaml
@@ -4,11 +4,16 @@ metadata:
   name: auth-machine
   namespace: {{ template "ocis.namespace" . }}
   labels:
+    app: auth-machine
+    ocis-metrics: enabled
     {{- include "ocis.labels" . | nindent 4 }}
 spec:
+  selector:
+    app: auth-machine
   ports:
     - name: grpc
       port: 9166
       protocol: TCP
-  selector:
-    app: auth-machine
+    - name: metrics-debug
+      port: 9167
+      protocol: TCP
diff --git a/charts/ocis/templates/frontend/deployment.yaml b/charts/ocis/templates/frontend/deployment.yaml
@@ -46,8 +46,14 @@ spec:
             - name: FRONTEND_LOG_PRETTY
               value: "{{ .Values.logging.pretty }}"
 
+            - name: FRONTEND_DEBUG_PPROF
+              value: "{{ .Values.debug.profiling }}"
+
             - name: FRONTEND_HTTP_ADDR
               value: 0.0.0.0:9140
+            - name: FRONTEND_DEBUG_ADDR
+              value: 0.0.0.0:9141
+
             - name: FRONTEND_PUBLIC_URL
               value: "https://{{ .Values.externalDomain }}"
 
@@ -78,10 +84,21 @@ spec:
                   name: {{ .Values.secretRefs.transferSecretSecretRef }}
                   key: transfer-secret
 
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics-debug
+            timeoutSeconds: 10
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            failureThreshold: 3
+
           resources: {{ toYaml .Values.resources | nindent 12 }}
           ports:
-          - name: http
-            containerPort: 9140
+            - name: http
+              containerPort: 9140
+            - name: metrics-debug
+              containerPort: 9141
           volumeMounts:
             - name: tmp-volume
               mountPath: /tmp

diff --git a/charts/ocis/templates/frontend/service.yaml b/charts/ocis/templates/frontend/service.yaml
@@ -4,11 +4,16 @@ metadata:
   name: frontend
   namespace: {{ template "ocis.namespace" . }}
   labels:
+    app: frontend
+    ocis-metrics: enabled
     {{- include "ocis.labels" . | nindent 4 }}
 spec:
+  selector:
+    app: frontend
   ports:
     - name: http
       port: 9140
       protocol: TCP
-  selector:
-    app: frontend
+    - name: metrics-debug
+      port: 9141
+      protocol: TCP
diff --git a/charts/ocis/templates/gateway/deployment.yaml b/charts/ocis/templates/gateway/deployment.yaml
@@ -46,8 +46,13 @@ spec:
             - name: GATEWAY_LOG_PRETTY
               value: "{{ .Values.logging.pretty }}"
 
+            - name: GATEWAY_DEBUG_PPROF
+              value: "{{ .Values.debug.profiling }}"
+
             - name: GATEWAY_GRPC_ADDR
               value: 0.0.0.0:9142
+            - name: GATEWAY_DEBUG_ADDR
+              value: 0.0.0.0:9143
 
             - name: REVA_GATEWAY
               # we also could have gateway:9142 here, but some
@@ -95,10 +100,21 @@ spec:
                   name: {{ .Values.secretRefs.transferSecretSecretRef }}
                   key: transfer-secret
 
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics-debug
+            timeoutSeconds: 10
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            failureThreshold: 3
+
           resources: {{ toYaml .Values.resources | nindent 12 }}
           ports:
             - name: grpc
               containerPort: 9142
+            - name: metrics-debug
+              containerPort: 9143
           volumeMounts:
             - name: tmp-volume
               mountPath: /tmp

diff --git a/charts/ocis/templates/gateway/service.yaml b/charts/ocis/templates/gateway/service.yaml
@@ -4,11 +4,16 @@ metadata:
   name: gateway
   namespace: {{ template "ocis.namespace" . }}
   labels:
+    app: gateway
+    ocis-metrics: enabled
     {{- include "ocis.labels" . | nindent 4 }}
 spec:
+  selector:
+    app: gateway
   ports:
     - name: grpc
       port: 9142
       protocol: TCP
-  selector:
-    app: gateway
+    - name: metrics-debug
+      port: 9143
+      protocol: TCP