From b21a8c01521c650c0b3eab90d3f1a36d136ebde1 Mon Sep 17 00:00:00 2001 From: acekingke Date: Thu, 20 Jan 2022 11:12:36 +0800 Subject: [PATCH] *: support the cronjob to backup #215 --- PROJECT | 8 + api/v1alpha1/mysqlcluster_types.go | 11 ++ api/v1alpha1/zz_generated.deepcopy.go | 5 + backup/cronbackup.go | 130 +++++++++++++++ .../crds/mysql.radondb.com_mysqlclusters.yaml | 8 + cmd/manager/main.go | 12 ++ .../mysql.radondb.com_mysqlclusters.yaml | 8 + ...ha1_mysqlcluster_backup_schedule_demo.yaml | 88 ++++++++++ controllers/backupcron_controller.go | 156 ++++++++++++++++++ docs/en-us/backupSchedule.md | 11 ++ go.mod | 1 + go.sum | 2 + 12 files changed, 440 insertions(+) create mode 100644 backup/cronbackup.go create mode 100644 config/samples/mysql_v1alpha1_mysqlcluster_backup_schedule_demo.yaml create mode 100644 controllers/backupcron_controller.go create mode 100644 docs/en-us/backupSchedule.md diff --git a/PROJECT b/PROJECT index 20c5e7e4..cc70a969 100644 --- a/PROJECT +++ b/PROJECT @@ -30,6 +30,14 @@ resources: kind: Backup path: github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: radondb.com + group: mysql + kind: BackupCron + version: v1alpha1 - api: crdVersion: v1 namespaced: true diff --git a/api/v1alpha1/mysqlcluster_types.go b/api/v1alpha1/mysqlcluster_types.go index 65e29934..6a8037f0 100644 --- a/api/v1alpha1/mysqlcluster_types.go +++ b/api/v1alpha1/mysqlcluster_types.go @@ -85,6 +85,17 @@ type MysqlClusterSpec struct { // Represents NFS ip address where cluster restore from. // +optional NFSServerAddress string `json:"nfsServerAddress,omitempty"` + + // Specify under crontab format interval to take backups + // leave it empty to deactivate the backup process + // Defaults to "" + // +optional + BackupSchedule string `json:"backupSchedule,omitempty"` + + // If set keeps last BackupScheduleJobsHistoryLimit Backups + // +optional + // +kubebuilder:default:=6 + BackupScheduleJobsHistoryLimit *int `json:"backupScheduleJobsHistoryLimit,omitempty"` } // MysqlOpts defines the options of MySQL container. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index df67da29..133dc422 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -263,6 +263,11 @@ func (in *MysqlClusterSpec) DeepCopyInto(out *MysqlClusterSpec) { in.MetricsOpts.DeepCopyInto(&out.MetricsOpts) in.PodPolicy.DeepCopyInto(&out.PodPolicy) in.Persistence.DeepCopyInto(&out.Persistence) + if in.BackupScheduleJobsHistoryLimit != nil { + in, out := &in.BackupScheduleJobsHistoryLimit, &out.BackupScheduleJobsHistoryLimit + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MysqlClusterSpec. diff --git a/backup/cronbackup.go b/backup/cronbackup.go new file mode 100644 index 00000000..1e44147a --- /dev/null +++ b/backup/cronbackup.go @@ -0,0 +1,130 @@ +package backup + +import ( + "context" + "fmt" + "sort" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/go-logr/logr" + apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" +) + +// The job structure contains the context to schedule a backup +type CronJob struct { + ClusterName string + Namespace string + + // kubernetes client + Client client.Client + + BackupScheduleJobsHistoryLimit *int + Image string + Log logr.Logger +} + +func (j *CronJob) Run() { + // nolint: govet + log := j.Log + log.Info("scheduled backup job started") + + // run garbage collector if needed + if j.BackupScheduleJobsHistoryLimit != nil { + defer j.backupGC() + } + + // check if a backup is running + if j.scheduledBackupsRunningCount() > 0 { + log.Info("at least a backup is running", "running_backups_count", j.scheduledBackupsRunningCount()) + return + } + + // create the backup + if _, err := j.createBackup(); err != nil { + log.Error(err, "failed to create backup") + } +} + +func (j *CronJob) scheduledBackupsRunningCount() int { + log := j.Log + backupsList := &apiv1alpha1.BackupList{} + // select all backups with labels recurrent=true and and not completed of the cluster + selector := j.backupSelector() + client.MatchingFields{"status.completed": "false"}.ApplyToList(selector) + + if err := j.Client.List(context.TODO(), backupsList, selector); err != nil { + log.Error(err, "failed getting backups", "selector", selector) + return 0 + } + + return len(backupsList.Items) +} + +func (j *CronJob) backupSelector() *client.ListOptions { + selector := &client.ListOptions{} + + client.InNamespace(j.Namespace).ApplyToList(selector) + client.MatchingLabels(j.recurrentBackupLabels()).ApplyToList(selector) + + return selector +} + +func (j *CronJob) recurrentBackupLabels() map[string]string { + return map[string]string{ + "recurrent": "true", + "cluster": j.ClusterName, + } +} + +func (j *CronJob) backupGC() { + var err error + log := j.Log + backupsList := &apiv1alpha1.BackupList{} + if err = j.Client.List(context.TODO(), backupsList, j.backupSelector()); err != nil { + log.Error(err, "failed getting backups", "selector", j.backupSelector()) + return + } + + // sort backups by creation time before removing extra backups + sort.Sort(byTimestamp(backupsList.Items)) + + for i, backup := range backupsList.Items { + if i >= *j.BackupScheduleJobsHistoryLimit { + // delete the backup + if err = j.Client.Delete(context.TODO(), &backup); err != nil { + log.Error(err, "failed to delete a backup", "backup", backup) + } + } + } +} + +func (j *CronJob) createBackup() (*apiv1alpha1.Backup, error) { + backupName := fmt.Sprintf("%s-auto-%s", j.ClusterName, time.Now().Format("2006-01-02t15-04-05")) + + backup := &apiv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: backupName, + Namespace: j.Namespace, + Labels: j.recurrentBackupLabels(), + }, + Spec: apiv1alpha1.BackupSpec{ + ClusterName: j.ClusterName, + //TODO modify to cluster sidecar image + Image: j.Image, + //RemoteDeletePolicy: j.BackupRemoteDeletePolicy, + HostName: fmt.Sprintf("%s-mysql-0", j.ClusterName), + }, + } + return backup, j.Client.Create(context.TODO(), backup) +} + +type byTimestamp []apiv1alpha1.Backup + +func (a byTimestamp) Len() int { return len(a) } +func (a byTimestamp) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byTimestamp) Less(i, j int) bool { + return a[j].ObjectMeta.CreationTimestamp.Before(&a[i].ObjectMeta.CreationTimestamp) +} diff --git a/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml b/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml index c338ba0e..de7fc65c 100644 --- a/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml +++ b/charts/mysql-operator/crds/mysql.radondb.com_mysqlclusters.yaml @@ -58,6 +58,14 @@ spec: spec: description: MysqlClusterSpec defines the desired state of MysqlCluster properties: + backupSchedule: + description: Specify under crontab format interval to take backups + leave it empty to deactivate the backup process Defaults to "" + type: string + backupScheduleJobsHistoryLimit: + default: 6 + description: If set keeps last BackupScheduleJobsHistoryLimit Backups + type: integer backupSecretName: description: Represents the name of the secret that contains credentials to connect to the storage provider to store backups. diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 40cad437..fdf1e8d3 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "os" + "sync" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -34,6 +35,7 @@ import ( mysqlv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" "github.com/radondb/radondb-mysql-kubernetes/controllers" "github.com/radondb/radondb-mysql-kubernetes/internal" + "github.com/wgliang/cron" //+kubebuilder:scaffold:imports ) @@ -116,6 +118,16 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "MysqlUser") os.Exit(1) } + if err = (&controllers.BackupCronReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("controller.BackupCron"), + Cron: cron.New(), + LockJobRegister: new(sync.Mutex), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "BackupCron") + os.Exit(1) + } //+kubebuilder:scaffold:builder if os.Getenv("ENABLE_WEBHOOKS") != "false" { if err = (&mysqlv1alpha1.MysqlCluster{}).SetupWebhookWithManager(mgr); err != nil { diff --git a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml index c338ba0e..de7fc65c 100644 --- a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml +++ b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml @@ -58,6 +58,14 @@ spec: spec: description: MysqlClusterSpec defines the desired state of MysqlCluster properties: + backupSchedule: + description: Specify under crontab format interval to take backups + leave it empty to deactivate the backup process Defaults to "" + type: string + backupScheduleJobsHistoryLimit: + default: 6 + description: If set keeps last BackupScheduleJobsHistoryLimit Backups + type: integer backupSecretName: description: Represents the name of the secret that contains credentials to connect to the storage provider to store backups. diff --git a/config/samples/mysql_v1alpha1_mysqlcluster_backup_schedule_demo.yaml b/config/samples/mysql_v1alpha1_mysqlcluster_backup_schedule_demo.yaml new file mode 100644 index 00000000..3eda928e --- /dev/null +++ b/config/samples/mysql_v1alpha1_mysqlcluster_backup_schedule_demo.yaml @@ -0,0 +1,88 @@ +apiVersion: mysql.radondb.com/v1alpha1 +kind: MysqlCluster +metadata: + name: sample +spec: + replicas: 3 + mysqlVersion: "5.7" + + # the backupSecretName specify the secret file name which store S3 information, + # if you want S3 backup or restore, please create backup_secret.yaml, uncomment below and fill secret name: + backupSecretName: sample-backup-secret + + # if you want create mysqlcluster from S3, uncomment and fill the directory in S3 bucket below: + # restoreFrom: + BackupSchedule: "0 50 * * * *" + mysqlOpts: + rootPassword: "RadonDB@123" + rootHost: localhost + user: radondb_usr + password: RadonDB@123 + database: radondb + initTokuDB: true + + # A simple map between string and string. + # Such as: + # mysqlConf: + # expire_logs_days: "7" + mysqlConf: {} + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + + xenonOpts: + image: radondb/xenon:1.1.5-alpha + admitDefeatHearbeatCount: 5 + electionTimeout: 10000 + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 100m + memory: 256Mi + + metricsOpts: + enabled: false + image: prom/mysqld-exporter:v0.12.1 + + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + + podPolicy: + imagePullPolicy: IfNotPresent + sidecarImage: radondb/mysql-sidecar:latest + busyboxImage: busybox:1.32 + + slowLogTail: false + auditLogTail: false + + labels: {} + annotations: {} + affinity: {} + priorityClassName: "" + tolerations: [] + schedulerName: "" + # extraResources defines quotas for containers other than mysql or xenon. + extraResources: + requests: + cpu: 10m + memory: 32Mi + + persistence: + enabled: true + accessModes: + - ReadWriteOnce + #storageClass: "" + size: 20Gi diff --git a/controllers/backupcron_controller.go b/controllers/backupcron_controller.go new file mode 100644 index 00000000..af3aad72 --- /dev/null +++ b/controllers/backupcron_controller.go @@ -0,0 +1,156 @@ +/* +Copyright 2021 RadonDB. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "reflect" + "sync" + + "github.com/go-logr/logr" + "github.com/radondb/radondb-mysql-kubernetes/mysqlcluster" + "github.com/wgliang/cron" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + apiv1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" + "github.com/radondb/radondb-mysql-kubernetes/backup" +) + +// BackupCronReconciler reconciles a BackupCron object +type BackupCronReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + Cron *cron.Cron + LockJobRegister *sync.Mutex +} + +type startStopCron struct { + Cron *cron.Cron +} + +func (c startStopCron) Start(ctx context.Context) error { + c.Cron.Start() + <-ctx.Done() + c.Cron.Stop() + + return nil +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the BackupCron object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile +func (r *BackupCronReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx).WithName("controllers").WithName("backupCronJob") + + instance := mysqlcluster.New(&apiv1alpha1.MysqlCluster{}) + + err := r.Get(ctx, req.NamespacedName, instance.Unwrap()) + if err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + log.Info("instance not found, maybe removed") + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if err = instance.Validate(); err != nil { + return ctrl.Result{}, err + } + // if spec.backupScheduler is not set then don't do anything + if len(instance.Spec.BackupSchedule) == 0 { + return reconcile.Result{}, nil + } + + schedule, err := cron.Parse(instance.Spec.BackupSchedule) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to parse schedule: %s", err) + } + + log.V(1).Info("register cluster in cronjob", "key", instance, "schedule", schedule) + + return ctrl.Result{}, r.updateClusterSchedule(ctx, instance.Unwrap(), schedule, log) +} + +// updateClusterSchedule creates/updates a cron job for specified cluster. +func (r *BackupCronReconciler) updateClusterSchedule(ctx context.Context, cluster *apiv1alpha1.MysqlCluster, schedule cron.Schedule, log logr.Logger) error { + + r.LockJobRegister.Lock() + defer r.LockJobRegister.Unlock() + + for _, entry := range r.Cron.Entries() { + j, ok := entry.Job.(*backup.CronJob) + if ok && j.ClusterName == cluster.Name && j.Namespace == cluster.Namespace { + log.V(1).Info("cluster already added to cron.", "key", cluster) + + // change scheduler for already added crons + if !reflect.DeepEqual(entry.Schedule, schedule) { + log.Info("update cluster scheduler", "key", cluster, + "scheduler", cluster.Spec.BackupSchedule) + + if err := r.Cron.Remove(cluster.Name); err != nil { + return err + } + break + } + if j.Image != cluster.Spec.PodPolicy.SidecarImage { + log.Info("update cluster image", "key", cluster, "image", cluster.Spec.PodPolicy.SidecarImage) + j.Image = cluster.Spec.PodPolicy.SidecarImage + } + return nil + } + } + + r.Cron.Schedule(schedule, &backup.CronJob{ + ClusterName: cluster.Name, + Namespace: cluster.Namespace, + Client: r.Client, + Image: cluster.Spec.PodPolicy.SidecarImage, + BackupScheduleJobsHistoryLimit: cluster.Spec.BackupScheduleJobsHistoryLimit, + //BackupRemoteDeletePolicy: cluster.Spec.BackupRemoteDeletePolicy, + Log: log, + }, cluster.Name) + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *BackupCronReconciler) SetupWithManager(mgr ctrl.Manager) error { + sscron := startStopCron{ + Cron: r.Cron, + } + mgr.Add(sscron) + return ctrl.NewControllerManagedBy(mgr). + // Uncomment the following line adding a pointer to an instance of the controlled resource as an argument + For(&apiv1alpha1.MysqlCluster{}). + Complete(r) +} diff --git a/docs/en-us/backupSchedule.md b/docs/en-us/backupSchedule.md new file mode 100644 index 00000000..837b5e70 --- /dev/null +++ b/docs/en-us/backupSchedule.md @@ -0,0 +1,11 @@ +backupSchedule: "0 0 0 * * *" # daily + +Crontab takes 6 arguments from the traditional 5. The additional argument is a seconds field. Some crontab examples and their predefined schedulers: + +| Entry | Equivalent To | Description | +| ------------- | ----- | ----------- | +| 15 0 0 1 1 * | @yearly (or @annually) | Run once a year, midnight, Jan. 1st, 15th second | +| 0 0 0 1 * * | @monthly | Run once a month, midnight, first of month, 0 second | +| 0 0 0 * * 0 | @weekly | Run once a week, midnight between Sat/Sun, 0 second | +| 0 0 0 * * * | @daily (or @midnight) | Run once a day, midnight, 0 second, 0 second | +| 0 0 * * * * | @hourly | Run once an hour, beginning of hour, 0 second | \ No newline at end of file diff --git a/go.mod b/go.mod index 372f55ea..b9c23126 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/presslabs/controller-util v0.3.0 github.com/spf13/cobra v1.1.3 github.com/stretchr/testify v1.7.0 + github.com/wgliang/cron v0.0.0-20210929064749-bba7232645e5 golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736 // indirect k8s.io/api v0.21.3 k8s.io/apimachinery v0.21.3 diff --git a/go.sum b/go.sum index 036e596d..1afc61ff 100644 --- a/go.sum +++ b/go.sum @@ -417,6 +417,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/wgliang/cron v0.0.0-20210929064749-bba7232645e5 h1:HRafmqrc2v44CoXo3aUUH1PrOUY0Ol6PyLJP4QzEBHs= +github.com/wgliang/cron v0.0.0-20210929064749-bba7232645e5/go.mod h1:8vrxYe6J+ZIHJViXE2UhdSbbu3VWHGxLo+QzdqeGDvM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=