From 64cce96d772e54ecf5a56c5fea36e26786f9b0fc Mon Sep 17 00:00:00 2001 From: Yaroslav Kasatikov Date: Tue, 11 Jul 2023 07:54:33 +0300 Subject: [PATCH 1/4] logical Backup initial --- .../templates/backup/logicalBackup.yaml | 142 ++++++++++++++++++ cnpg-cluster-helm-chart/values.yaml | 5 + 2 files changed, 147 insertions(+) create mode 100644 cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml diff --git a/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml b/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml new file mode 100644 index 0000000..ac23e02 --- /dev/null +++ b/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml @@ -0,0 +1,142 @@ +{{- if .Values.logicalBackup.enabled }} +{{- if .Values.logicalBackup.createBucket -}} +apiVersion: s3.advatys.com/v1alpha1 +kind: Bucket +metadata: + annotations: + argocd.argoproj.io/sync-wave: "-5" + name: {{ include "cnpg-cluster.fullname" . }}-backup-logical + labels: + {{- include "cnpg-cluster.labels" . | nindent 4 }} +spec: + region: {{ .Values.backup.bucket.region }} + bucketClassName: {{ .Values.backup.bucket.class }} + bucketPreventDestroy: true + roles: + - actions: + - "PutObject" + - "GetObject" + - "DeleteObject" + - "ListBucket" + resources: + - "/*" + - "" + + subjects: + - type: KubernetesServiceAccount + cluster: {{ .Values.kubeClusterName }} + name: {{ include "cnpg-cluster.fullname" . }}-backup + namespace: {{ .Release.Namespace }} + effect: Allow + name: {{ include "cnpg-cluster.fullname" . }}-backup-creator + {{- if eq .Values.bootstrap.type "physical" }} + - actions: + - "GetObject" + - "ListBucketMultipartUploads" + - "ListBucket" + - "ListMultipartUploadParts" + - "GetObjectVersion" + resources: + - "/*" + - "" + + subjects: + - type: KubernetesServiceAccount + cluster: {{ .Values.kubeClusterName }} + name: {{ include "cnpg-cluster.fullname" . }}-restore + namespace: {{ .Release.Namespace }} + effect: Allow + name: {{ include "cnpg-cluster.fullname" . }}-backup-reader + {{- end }} +{{- end }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "cnpg-cluster.fullname" . }}-backup-logical + labels: + {{- include "cnpg-cluster.labels" . | nindent 4 }} +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 5 + + jobTemplate: + spec: + backoffLimit: 0 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/auth-path: "auth/{{ .Values.backup.bucket.vault.authPath }}" + vault.hashicorp.com/auth-type: "jwt" + vault.hashicorp.com/auth-config-remove-jwt-after-reading: 'false' + vault.hashicorp.com/agent-pre-populate-only : "true" + vault.hashicorp.com/auth-config-path: /var/run/secrets/tokens/vault-jwt-token + vault.hashicorp.com/agent-inject-secret-secret.yaml: "{{ .Values.backup.bucket.vault.awsBackend }}/creds/{{ include "cnpg-cluster.fullname" . }}-backup-creator" + vault.hashicorp.com/agent-inject-template-secret.yaml: | + {{ "{{-" }} with secret "{{ .Values.backup.bucket.vault.awsBackend }}/creds/{{ include "cnpg-cluster.fullname" . }}-backup-creator" {{ "-}}" }} + apiVersion: v1 + stringData: + ACCESS_KEY_ID: {{ "{{" }} .Data.access_key {{ "}}" }} + ACCESS_SECRET_KEY: {{ "{{" }} .Data.secret_key {{ "}}" }} + ACCESS_SESSION_TOKEN: {{ "{{" }} .Data.security_token {{ "}}" }} + kind: Secret + metadata: + name: {{ include "cnpg-cluster.fullname" . }}-backup-credentials + namespace: {{ .Release.Namespace }} + {{ "{{" }} end {{ "}}" }} + vault.hashicorp.com/role: "{{ include "cnpg-cluster.fullname" . }}-backup-creator-{{ include "cnpg-cluster.fullname" . }}-backup-{{ .Release.Namespace }}" + vault.hashicorp.com/secret-volume-path-secret.yaml: /kubesecret + vault.hashicorp.com/service: "{{ .Values.backup.bucket.vault.address }}" + vault.hashicorp.com/tls-skip-verify: "false" + vault.hashicorp.com/agent-copy-volume-mounts: "get-secret" + vault.hashicorp.com/ca-cert: "/usr/local/share/ca-certificates/ca-bundle.crt" + spec: + restartPolicy: Never + serviceAccountName: {{ include "cnpg-cluster.fullname" . }}-backup + volumes: + - name: custom-trusted-ca + configMap: + name: custom-trusted-ca + - name: vault-jwt-token + projected: + sources: + - serviceAccountToken: + path: vault-jwt-token + expirationSeconds: 600 + audience: "{{ .Values.backup.bucket.vault.address }}" + - name: backup + emptyDir: {} + containers: + - name: logical-backup + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "IfNotPresent" + env: + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ include "cnpg-cluster.fullname" . }}-superuser + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "cnpg-cluster.fullname" . }}-superuser + key: password + + volumeMounts: + - mountPath: /var/run/secrets/tokens + name: vault-jwt-token + - name: custom-trusted-ca + mountPath: /usr/local/share/ca-certificates + readOnly: true + - name: backup + mountPath: /backup + + command: + - sh + - -c + - "pg_dumpall -h {{ include "cnpg-cluster.fullname" . }}-ro | gzip > /backup/dump_all_{{ include "cnpg-cluster.fullname" . }}`_date +%F-%T|sed 's/:/-/g'.sql`" + + schedule: '{{ .Values.logicalBackup.schedule }}' + successfulJobsHistoryLimit: 3 +{{- end }} diff --git a/cnpg-cluster-helm-chart/values.yaml b/cnpg-cluster-helm-chart/values.yaml index 2a60a0d..8ce892a 100644 --- a/cnpg-cluster-helm-chart/values.yaml +++ b/cnpg-cluster-helm-chart/values.yaml @@ -44,6 +44,11 @@ postgresql: {} clusterReplication: false +logicalBackup: + enabled: true + createBucket: true + schedule: "00 1 * * *" + backup: # Options: # - none From f2765fd236bfcb5d01c4940a0e25feb64175cd93 Mon Sep 17 00:00:00 2001 From: Yaroslav Kasatikov Date: Tue, 11 Jul 2023 08:06:00 +0300 Subject: [PATCH 2/4] Logical --- .../templates/backup/logicalBackup.yaml | 52 +------------------ .../templates/backup/scheduledBackup.yaml | 15 ------ 2 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 cnpg-cluster-helm-chart/templates/backup/scheduledBackup.yaml diff --git a/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml b/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml index ac23e02..885c06a 100644 --- a/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml +++ b/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml @@ -1,54 +1,4 @@ -{{- if .Values.logicalBackup.enabled }} -{{- if .Values.logicalBackup.createBucket -}} -apiVersion: s3.advatys.com/v1alpha1 -kind: Bucket -metadata: - annotations: - argocd.argoproj.io/sync-wave: "-5" - name: {{ include "cnpg-cluster.fullname" . }}-backup-logical - labels: - {{- include "cnpg-cluster.labels" . | nindent 4 }} -spec: - region: {{ .Values.backup.bucket.region }} - bucketClassName: {{ .Values.backup.bucket.class }} - bucketPreventDestroy: true - roles: - - actions: - - "PutObject" - - "GetObject" - - "DeleteObject" - - "ListBucket" - resources: - - "/*" - - "" - - subjects: - - type: KubernetesServiceAccount - cluster: {{ .Values.kubeClusterName }} - name: {{ include "cnpg-cluster.fullname" . }}-backup - namespace: {{ .Release.Namespace }} - effect: Allow - name: {{ include "cnpg-cluster.fullname" . }}-backup-creator - {{- if eq .Values.bootstrap.type "physical" }} - - actions: - - "GetObject" - - "ListBucketMultipartUploads" - - "ListBucket" - - "ListMultipartUploadParts" - - "GetObjectVersion" - resources: - - "/*" - - "" - - subjects: - - type: KubernetesServiceAccount - cluster: {{ .Values.kubeClusterName }} - name: {{ include "cnpg-cluster.fullname" . }}-restore - namespace: {{ .Release.Namespace }} - effect: Allow - name: {{ include "cnpg-cluster.fullname" . }}-backup-reader - {{- end }} -{{- end }} +{{- if eq .Values.backup.type "logical" }} --- apiVersion: batch/v1 kind: CronJob diff --git a/cnpg-cluster-helm-chart/templates/backup/scheduledBackup.yaml b/cnpg-cluster-helm-chart/templates/backup/scheduledBackup.yaml deleted file mode 100644 index 9855d6d..0000000 --- a/cnpg-cluster-helm-chart/templates/backup/scheduledBackup.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if eq .Values.backup.type "physical" }} -apiVersion: postgresql.cnpg.io/v1 -kind: ScheduledBackup -metadata: - name: {{ include "cnpg-cluster.fullname" . }} -spec: - schedule: "{{ .Values.backup.schedule }}" - - #backupOwnerReference indicates which ownerReference should be put inside the created backup resources. - backupOwnerReference: none - - cluster: - name: {{ include "cnpg-cluster.fullname" . }} - -{{- end }} From b4d791100647224f0718fd768012d3475ffb1179 Mon Sep 17 00:00:00 2001 From: Yaroslav Kasatikov Date: Tue, 11 Jul 2023 09:38:38 +0300 Subject: [PATCH 3/4] Logical --- .../templates/backup/physicalBackup.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml diff --git a/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml b/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml new file mode 100644 index 0000000..9855d6d --- /dev/null +++ b/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml @@ -0,0 +1,15 @@ +{{- if eq .Values.backup.type "physical" }} +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: {{ include "cnpg-cluster.fullname" . }} +spec: + schedule: "{{ .Values.backup.schedule }}" + + #backupOwnerReference indicates which ownerReference should be put inside the created backup resources. + backupOwnerReference: none + + cluster: + name: {{ include "cnpg-cluster.fullname" . }} + +{{- end }} From b385b7aabc20db2b5839a932a06b7dd9e7f71fe7 Mon Sep 17 00:00:00 2001 From: Yaroslav Kasatikov Date: Tue, 11 Jul 2023 11:39:22 +0300 Subject: [PATCH 4/4] Logical backup + monitoring --- .../templates/backup/logicalBackup.yaml | 49 +++++++++++++------ .../templates/backup/physicalBackup.yaml | 2 +- .../monitoring/backupMonitoring.yaml | 21 ++++++++ cnpg-cluster-helm-chart/values.yaml | 6 +-- 4 files changed, 56 insertions(+), 22 deletions(-) create mode 100644 cnpg-cluster-helm-chart/templates/monitoring/backupMonitoring.yaml diff --git a/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml b/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml index 885c06a..1270737 100644 --- a/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml +++ b/cnpg-cluster-helm-chart/templates/backup/logicalBackup.yaml @@ -24,19 +24,14 @@ spec: vault.hashicorp.com/auth-config-path: /var/run/secrets/tokens/vault-jwt-token vault.hashicorp.com/agent-inject-secret-secret.yaml: "{{ .Values.backup.bucket.vault.awsBackend }}/creds/{{ include "cnpg-cluster.fullname" . }}-backup-creator" vault.hashicorp.com/agent-inject-template-secret.yaml: | - {{ "{{-" }} with secret "{{ .Values.backup.bucket.vault.awsBackend }}/creds/{{ include "cnpg-cluster.fullname" . }}-backup-creator" {{ "-}}" }} - apiVersion: v1 - stringData: - ACCESS_KEY_ID: {{ "{{" }} .Data.access_key {{ "}}" }} - ACCESS_SECRET_KEY: {{ "{{" }} .Data.secret_key {{ "}}" }} - ACCESS_SESSION_TOKEN: {{ "{{" }} .Data.security_token {{ "}}" }} - kind: Secret - metadata: - name: {{ include "cnpg-cluster.fullname" . }}-backup-credentials - namespace: {{ .Release.Namespace }} - {{ "{{" }} end {{ "}}" }} + {{ "{{-" }} with secret "/tmp/.aws/credentials {{ "}}" }} + [default] + aws_access_key_id={{ "{{" }} .Data.access_key {{ "}}" }} + aws_secret_access_key={{ "{{" }} .Data.secret_key {{ "}}" }} + aws_session_token={{ "{{" }} .Data.security_token {{ "}}" }} + {{ "{{" }} end {{ "}}" }} vault.hashicorp.com/role: "{{ include "cnpg-cluster.fullname" . }}-backup-creator-{{ include "cnpg-cluster.fullname" . }}-backup-{{ .Release.Namespace }}" - vault.hashicorp.com/secret-volume-path-secret.yaml: /kubesecret + vault.hashicorp.com/secret-volume-path-secret.yaml: /tmp/.aws/ vault.hashicorp.com/service: "{{ .Values.backup.bucket.vault.address }}" vault.hashicorp.com/tls-skip-verify: "false" vault.hashicorp.com/agent-copy-volume-mounts: "get-secret" @@ -57,7 +52,7 @@ spec: audience: "{{ .Values.backup.bucket.vault.address }}" - name: backup emptyDir: {} - containers: + initContainers: - name: logical-backup image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: "IfNotPresent" @@ -81,12 +76,34 @@ spec: readOnly: true - name: backup mountPath: /backup - command: - sh - -c - - "pg_dumpall -h {{ include "cnpg-cluster.fullname" . }}-ro | gzip > /backup/dump_all_{{ include "cnpg-cluster.fullname" . }}`_date +%F-%T|sed 's/:/-/g'.sql`" + - "pg_dumpall -h {{ include "cnpg-cluster.fullname" . }}-ro | gzip > /backup/dump_all_{{ include "cnpg-cluster.fullname" . }}_`date +%F-%T|sed 's/:/-/g'`.sql.gz" + containers: + - command: + - sh + - -c + - aws s3 sync /backup/ s3://{{ include "cnpg-cluster.fullname" . }}-backup/ + image: amazon/aws-cli:2.13.0 + imagePullPolicy: IfNotPresent + env: + - name: HOME + value: /tmp + name: aws-backup + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/run/secrets/tokens + name: vault-jwt-token + - mountPath: /usr/local/share/ca-certificates + name: custom-trusted-ca + readOnly: true + - mountPath: /backup + name: backup + - schedule: '{{ .Values.logicalBackup.schedule }}' + schedule: '{{ .Values.backup.schedule }}' successfulJobsHistoryLimit: 3 {{- end }} diff --git a/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml b/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml index 9855d6d..5668bcd 100644 --- a/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml +++ b/cnpg-cluster-helm-chart/templates/backup/physicalBackup.yaml @@ -4,7 +4,7 @@ kind: ScheduledBackup metadata: name: {{ include "cnpg-cluster.fullname" . }} spec: - schedule: "{{ .Values.backup.schedule }}" + schedule: "0 {{ .Values.backup.schedule }}" #backupOwnerReference indicates which ownerReference should be put inside the created backup resources. backupOwnerReference: none diff --git a/cnpg-cluster-helm-chart/templates/monitoring/backupMonitoring.yaml b/cnpg-cluster-helm-chart/templates/monitoring/backupMonitoring.yaml new file mode 100644 index 0000000..148e04b --- /dev/null +++ b/cnpg-cluster-helm-chart/templates/monitoring/backupMonitoring.yaml @@ -0,0 +1,21 @@ +{{- if eq .Values.backup.type "logical" }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "cnpg-cluster.fullname" . }}-backuplogical-alerts + labels: + openshift.io/prometheus-rule-evaluation-scope: leaf-prometheus +spec: + groups: + - name: backup-default.rules + rules: + - alert: backupJobFailed + annotations: + description: Job {{ "{{" }}$labels.job_name{{ "}}" }} is failed + summary: Logical Backup is Failed + expr: |- + kube_job_failed{namespace="{{ .Release.Namespace }}",job_name=~"{{ include "cnpg-cluster.fullname" . }}-backup-logical.*"} == 1 + for: 5m + labels: + severity: critical +{{- end }} diff --git a/cnpg-cluster-helm-chart/values.yaml b/cnpg-cluster-helm-chart/values.yaml index 8ce892a..9edf671 100644 --- a/cnpg-cluster-helm-chart/values.yaml +++ b/cnpg-cluster-helm-chart/values.yaml @@ -44,10 +44,6 @@ postgresql: {} clusterReplication: false -logicalBackup: - enabled: true - createBucket: true - schedule: "00 1 * * *" backup: # Options: @@ -55,7 +51,7 @@ backup: # - physical # - logical type: "none" - schedule: "0 */10 * * * *" # Required seconds!!! + schedule: "0 1 * * *" # NOT Required seconds anymore!!! bucket: endpointURL: "" dbClusterName: ""