From 713e6fbe7fc0f5a9e12cd031ab9f824c80359c82 Mon Sep 17 00:00:00 2001 From: buzhimingyonghu <42060366+buzhimingyonghu@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:08:08 +0800 Subject: [PATCH] feat:finish backup and restore (#2968) * backup and restore * delete sh * modify config --- tools/kubeblocks_helm/BackupRepo_config | 9 +++ tools/kubeblocks_helm/README.md | 15 +++++ .../{reinstall.sh => install.sh} | 2 - tools/kubeblocks_helm/install_ms.sh | 1 + tools/kubeblocks_helm/installdebug.sh | 4 -- .../templates/role.yaml | 17 ++++- .../dataprotection/backup.sh | 39 +++++++++++ .../dataprotection/restore.sh | 21 ++++++ .../templates/backupactionset.yaml | 37 +++++++++++ .../templates/backuppolicytemplate.yaml | 26 ++++++++ .../templates/componentdefinition-pika.yaml | 64 +++++++++++++------ tools/kubeblocks_helm/uninstall.sh | 1 + tools/kubeblocks_helm/uninstall_ms.sh | 1 + 13 files changed, 209 insertions(+), 28 deletions(-) create mode 100644 tools/kubeblocks_helm/BackupRepo_config rename tools/kubeblocks_helm/{reinstall.sh => install.sh} (53%) create mode 100644 tools/kubeblocks_helm/install_ms.sh delete mode 100644 tools/kubeblocks_helm/installdebug.sh create mode 100644 tools/kubeblocks_helm/pika-master-slave/dataprotection/backup.sh create mode 100644 tools/kubeblocks_helm/pika-master-slave/dataprotection/restore.sh create mode 100644 tools/kubeblocks_helm/pika-master-slave/templates/backupactionset.yaml create mode 100644 tools/kubeblocks_helm/pika-master-slave/templates/backuppolicytemplate.yaml create mode 100644 tools/kubeblocks_helm/uninstall.sh create mode 100644 tools/kubeblocks_helm/uninstall_ms.sh diff --git a/tools/kubeblocks_helm/BackupRepo_config b/tools/kubeblocks_helm/BackupRepo_config new file mode 100644 index 0000000000..852731c80f --- /dev/null +++ b/tools/kubeblocks_helm/BackupRepo_config @@ -0,0 +1,9 @@ +kbcli backuprepo create my-repo \ + --provider s3 \ + --region ${S3_REGION} \ + --bucket ${S3_BUCKET} \ + --endpoint ${S3_ENDPOINT} \ + --access-key-id ${S3_ACCESS_KEY} \ + --secret-access-key ${S3_SECRET_KEY} \ + --access-method ${S3_ACCESS_METHOD} \ + --default \ No newline at end of file diff --git a/tools/kubeblocks_helm/README.md b/tools/kubeblocks_helm/README.md index c7f8677fb5..6bbf142885 100644 --- a/tools/kubeblocks_helm/README.md +++ b/tools/kubeblocks_helm/README.md @@ -76,4 +76,19 @@ redis-cli -p 9221 ```bash helm uninstall pika-master-slave-cluster helm uninstall pika-master-slave +``` + +### Back up and restore a cluster +Ensure that the default BackupRepo is defined +Fellow the kubeblock docs [kubeblocks](https://www.kubeblocks.io/docs/preview/user_docs/maintenance/backup-and-restore/backup/backup-repo) + +create backup +```bash +kbcli cluster backup pika-master-slave-cluster --method datafile +``` + +Select a backup and create a cluster. + +```bash +kbcli cluster restore --backup ``` \ No newline at end of file diff --git a/tools/kubeblocks_helm/reinstall.sh b/tools/kubeblocks_helm/install.sh similarity index 53% rename from tools/kubeblocks_helm/reinstall.sh rename to tools/kubeblocks_helm/install.sh index 6671724766..45cc96d3c6 100644 --- a/tools/kubeblocks_helm/reinstall.sh +++ b/tools/kubeblocks_helm/install.sh @@ -1,3 +1 @@ -helm uninstall pika && helm uninstall pika-cluster -sleep 5 helm install pika ./pika && helm install pika-cluster ./pika-cluster \ No newline at end of file diff --git a/tools/kubeblocks_helm/install_ms.sh b/tools/kubeblocks_helm/install_ms.sh new file mode 100644 index 0000000000..aabf994979 --- /dev/null +++ b/tools/kubeblocks_helm/install_ms.sh @@ -0,0 +1 @@ +helm install pika-master-slave ./pika-master-slave && helm install pika-master-slave-cluster ./pika-master-slave-cluster \ No newline at end of file diff --git a/tools/kubeblocks_helm/installdebug.sh b/tools/kubeblocks_helm/installdebug.sh deleted file mode 100644 index 53a496ad62..0000000000 --- a/tools/kubeblocks_helm/installdebug.sh +++ /dev/null @@ -1,4 +0,0 @@ -# helm install pika ./pika && helm install pika-cluster ./pika-cluster - -helm template pika ./pika --output-dir ./output/pika -#helm template pika-cluster ./pika-cluster --output-dir ./output/pika-cluster \ No newline at end of file diff --git a/tools/kubeblocks_helm/pika-master-slave-cluster/templates/role.yaml b/tools/kubeblocks_helm/pika-master-slave-cluster/templates/role.yaml index 4792625f6a..13d3153d8b 100644 --- a/tools/kubeblocks_helm/pika-master-slave-cluster/templates/role.yaml +++ b/tools/kubeblocks_helm/pika-master-slave-cluster/templates/role.yaml @@ -6,9 +6,22 @@ metadata: labels: {{ include "pika-cluster.labels" . | nindent 4 }} rules: - - apiGroups: - - "" + - apiGroups: [""] resources: - events verbs: - create + - apiGroups: ["apps.kubeblocks.io"] + resources: + - clusters + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch diff --git a/tools/kubeblocks_helm/pika-master-slave/dataprotection/backup.sh b/tools/kubeblocks_helm/pika-master-slave/dataprotection/backup.sh new file mode 100644 index 0000000000..864e415d76 --- /dev/null +++ b/tools/kubeblocks_helm/pika-master-slave/dataprotection/backup.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -o pipefail +function handle_exit() { + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "failed with exit code $exit_code" + exit 1 + fi +} +trap handle_exit EXIT +export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" + +connect_url="redis-cli -h ${DP_DB_HOST} -p ${DP_DB_PORT}" +last_save=$(${connect_url} LASTSAVE) + +echo "INFO: start BGSAVE" +${connect_url} BGSAVE + +echo "INFO: wait for saving dump successfully" +while true; do + end_save=$(${connect_url} LASTSAVE) + if [ $end_save -ne $last_save ];then + break + fi + sleep 1 +done + +cd ${DATA_DIR} + +if [ -d "log" ] || [ -d "db" ]; then + tar -cvf - ./log ./db | datasafed push -z zstd-fastest - "${DP_BACKUP_NAME}.tar.zst" +else + echo "no log db" + exit 1 +fi +echo "INFO: save data file successfully" +TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}') +echo "{\"totalSize\":\"$TOTAL_SIZE\"}" >"${DP_BACKUP_INFO_FILE}" && sync diff --git a/tools/kubeblocks_helm/pika-master-slave/dataprotection/restore.sh b/tools/kubeblocks_helm/pika-master-slave/dataprotection/restore.sh new file mode 100644 index 0000000000..fd84f52b64 --- /dev/null +++ b/tools/kubeblocks_helm/pika-master-slave/dataprotection/restore.sh @@ -0,0 +1,21 @@ +set -e +set -o pipefail +export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" +mkdir -p ${DATA_DIR} +res=`find ${DATA_DIR} -type f` +data_protection_file=${DATA_DIR}/.kb-data-protection +if [ ! -z "${res}" ] && [ ! -f ${data_protection_file} ]; then + echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." + exit 1 +fi +# touch placeholder file +touch ${data_protection_file} + +backupFile="${DP_BACKUP_NAME}.tar.zst" +if [ "$(datasafed list ${backupFile})" == "${backupFile}" ]; then + datasafed pull -d zstd-fastest "${backupFile}" - | tar -xvf - -C ${DATA_DIR} +else + datasafed pull "${DP_BACKUP_NAME}.tar.gz" - | tar -xzvf - -C ${DATA_DIR} +fi +rm -rf ${data_protection_file} && sync \ No newline at end of file diff --git a/tools/kubeblocks_helm/pika-master-slave/templates/backupactionset.yaml b/tools/kubeblocks_helm/pika-master-slave/templates/backupactionset.yaml new file mode 100644 index 0000000000..694dec72a8 --- /dev/null +++ b/tools/kubeblocks_helm/pika-master-slave/templates/backupactionset.yaml @@ -0,0 +1,37 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: ActionSet +metadata: + name: pika-backup + labels: + clusterdefinition.kubeblocks.io/name: pika-master-slave-cluster + {{- include "pika.labels" . | nindent 4 }} +spec: + backupType: Full + env: + - name: DATA_DIR + value: /data + - name: DP_DB_PORT + value: "9221" + backup: + preBackup: [] + postBackup: [] + backupData: + image: {{ include "redis.image" . }} + runOnTargetPodNode: true + syncProgress: + enabled: true + intervalSeconds: 5 + command: + - bash + - -c + - | + {{- .Files.Get "dataprotection/backup.sh" | nindent 8 }} + restore: + prepareData: + image: {{ include "redis.image" . }} + command: + - bash + - -c + - | + {{- .Files.Get "dataprotection/restore.sh" | nindent 8 }} + postReady: [] \ No newline at end of file diff --git a/tools/kubeblocks_helm/pika-master-slave/templates/backuppolicytemplate.yaml b/tools/kubeblocks_helm/pika-master-slave/templates/backuppolicytemplate.yaml new file mode 100644 index 0000000000..2393ec3954 --- /dev/null +++ b/tools/kubeblocks_helm/pika-master-slave/templates/backuppolicytemplate.yaml @@ -0,0 +1,26 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: pika-backup-policy-template + labels: + clusterdefinition.kubeblocks.io/name: pika-master-slave-cluster + {{- include "pika.labels" . | nindent 4 }} +spec: + backupPolicies: + - componentDefs: + - pika + target: + role: master + backupMethods: + - name: datafile + snapshotVolumes: false + actionSetName: pika-backup + targetVolumes: + volumeMounts: + - name: data + mountPath: /data + schedules: + - backupMethod: datafile + enabled: true + cronExpression: "0 18 * * 0" + retentionPeriod: 3m diff --git a/tools/kubeblocks_helm/pika-master-slave/templates/componentdefinition-pika.yaml b/tools/kubeblocks_helm/pika-master-slave/templates/componentdefinition-pika.yaml index 243fcd84b1..cfeb7d111b 100644 --- a/tools/kubeblocks_helm/pika-master-slave/templates/componentdefinition-pika.yaml +++ b/tools/kubeblocks_helm/pika-master-slave/templates/componentdefinition-pika.yaml @@ -25,30 +25,32 @@ spec: volumeName: config volumes: - name: data + roles: + - name: master + serviceable: true + writable: true + - name: slave + serviceable: false + writable: false lifecycleActions: - postProvision: + roleProbe: + builtinHandler: custom customHandler: image: {{ include "redis.image" .}} exec: command: - - /bin/bash - - -c - - | - IFS=',' read -r -a pod_name_array <<< ${KB_CLUSTER_COMPONENT_POD_NAME_LIST} - # Ergodic name array - headless=".pika-master-slave-cluster-pika-headless" - for i in "${!pod_name_array[@]}"; do - pod_name=${pod_name_array[${i}]} - # Extracting the digital portion of the pod - pod_number=$(echo "${pod_name}" | grep -o '[0-9]*') - # If the pod number is greater than 0 - if (( pod_number > 0 )); then - pod0_svc=${pod_name_array[0]}${headless} - echo "exec:redis-cli -h ${pod_name}${headless} -p 9221 slaveof ${pod0_svc}:9221" - redis-cli -h "${pod_name}${headless}" -p 9221 slaveof "${pod0_svc}" 9221 - fi - done - preCondition: ComponentReady + - /bin/bash + - -c + - | + replication_info=$(redis-cli -h 127.0.0.1 -p 9221 info replication) + role=$(echo "${replication_info}" | awk -F':' '/^role:/ {print $2}' | tr -d '[:space:]') + if [ "$role"x = "master"x ]; then + echo -n "master" + else + echo -n "slave" + fi + periodSeconds: 1 + timeoutSeconds: 1 runtime: initContainers: - name: init-config @@ -86,4 +88,26 @@ spec: - "/pika/bin/pika" args: - "-c" - - "/data/pika.conf" \ No newline at end of file + - "/data/pika.conf" + - name: redis-cli-runner + image: {{ include "redis.image" . }} + imagePullPolicy: {{ include "pika.imagePullPolicy" . }} + command: + - /bin/bash + - -c + - | + while true; do + hostname=$KB_POD_NAME + if [[ "$hostname" =~ -([0-9]+)$ ]]; then + ordinal=${BASH_REMATCH[1]} + if [[ $ordinal -ge 1 && $ordinal -le 9 ]]; then + replication_info=$(redis-cli -h 127.0.0.1 -p 9221 info replication) + role=$(echo "${replication_info}" | awk -F':' '/^role:/ {print $2}' | tr -d '[:space:]') + if [ "$role"x = "master"x ]; then + redis-cli -h 127.0.0.1 -p 9221 slaveof pika-master-slave-cluster-pika-0.pika-master-slave-cluster-pika-headless 9221 + fi + fi + fi + sleep 10 + done + diff --git a/tools/kubeblocks_helm/uninstall.sh b/tools/kubeblocks_helm/uninstall.sh new file mode 100644 index 0000000000..bb043519c2 --- /dev/null +++ b/tools/kubeblocks_helm/uninstall.sh @@ -0,0 +1 @@ +helm uninstall pika-cluster && helm uninstall pika \ No newline at end of file diff --git a/tools/kubeblocks_helm/uninstall_ms.sh b/tools/kubeblocks_helm/uninstall_ms.sh new file mode 100644 index 0000000000..5f73287e1e --- /dev/null +++ b/tools/kubeblocks_helm/uninstall_ms.sh @@ -0,0 +1 @@ +helm uninstall pika-master-slave-cluster && helm uninstall pika-master-slave \ No newline at end of file