Skip to content

Commit

Permalink
*: add replica lag parameter
Browse files Browse the repository at this point in the history
  • Loading branch information
acekingke committed Aug 4, 2023
1 parent 2bf5fc1 commit 511a4a0
Show file tree
Hide file tree
Showing 12 changed files with 116 additions and 20 deletions.
2 changes: 2 additions & 0 deletions api/v1alpha1/mysqlcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ type MysqlClusterSpec struct {
// Readonlys Info.
// +optional
ReadOnlys *ReadOnlyType `json:"readonlys,omitempty"`
// Lagged
ReplicaLag *int32 `json:"lag,omitempty"`
// The number of pods from that set that must still be available after the
// eviction, even in the absence of the evicted pod
// +optional
Expand Down
5 changes: 5 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions api/v1beta1/mysqlcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ type MysqlClusterSpec struct {
// Readonlys Info.
// +optional
ReadOnlys *ReadOnlyType `json:"readonlys,omitempty"`
// Lagged
ReplicaLag *int32 `json:"lag,omitempty"`

// Username of new user to create.
// Only be a combination of letters, numbers or underlines. The length can not exceed 26 characters.
// +optional
Expand Down
2 changes: 2 additions & 0 deletions api/v1beta1/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ spec:
s3Schedule:
type: string
type: object
lag:
description: Lagged
format: int32
type: integer
metricsOpts:
default:
enabled: false
Expand Down Expand Up @@ -3526,6 +3530,10 @@ spec:
- Never
- IfNotPresent
type: string
lag:
description: Lagged
format: int32
type: integer
logOpts:
description: LogOpts is the options of log settings.
properties:
Expand Down
8 changes: 8 additions & 0 deletions config/crd/bases/mysql.radondb.com_mysqlclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ spec:
s3Schedule:
type: string
type: object
lag:
description: Lagged
format: int32
type: integer
metricsOpts:
default:
enabled: false
Expand Down Expand Up @@ -3508,6 +3512,10 @@ spec:
- Never
- IfNotPresent
type: string
lag:
description: Lagged
format: int32
type: integer
logOpts:
description: LogOpts is the options of log settings.
properties:
Expand Down
13 changes: 8 additions & 5 deletions config/samples/mysql_v1beta1_backup.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ kind: Backup
metadata:
name: backup-sample
spec:
# Add fields here
image: radondb/mysql57-sidecar:v3.0.0
# hostfrom if empty, use the leader as hostfrom
hostfrom: sample-mysql-0
backupops:
s3:
secretName: sample-backup-secret
clusterName: sample
# nfsServerAddress: ""
method: xtrabackup
# schedule:
# cronExpression: "*/2 * * * *"
# type: s3

72 changes: 64 additions & 8 deletions config/samples/mysql_v1beta1_mysqlcluster.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,68 @@
apiVersion: mysql.radondb.com/v1beta1
kind: MysqlCluster
metadata:
labels:
app.kubernetes.io/name: mysqlcluster
app.kubernetes.io/instance: mysqlcluster-sample
app.kubernetes.io/part-of: radondb-mysql-kubernetes
app.kubernetes.io/managed-by: kustomize
app.kubernetes.io/created-by: radondb-mysql-kubernetes
name: mysqlcluster-sample
name: sample
spec:
# TODO(user): Add fields here
backupOpts:
image: radondb/mysql80-sidecar:v3.0.0
resources:
requests:
cpu: 10m
memory: 32Mi
customTLSSecret: {}
dataSource:
S3backup:
name: ""
secretName: ""
remote: {}
image: percona/percona-server:8.0.25
imagePullPolicy: Always
logOpts:
resources:
requests:
cpu: 10m
memory: 32Mi
maxLagTime: 30
minAvailable: 50%
monitoringSpec:
exporter:
enabled: true
image: prom/mysqld-exporter:v0.12.1
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 10m
memory: 32Mi
mysqlConfig: {}
mysqlVersion: "8.0"
replicas: 3
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 100m
memory: 256Mi
storage:
accessModes:
- ReadWriteOnce
storageClassName: local-path
resources:
requests:
storage: 20Gi
user: radondb_usr
xenonOpts:
admitDefeatHearbeatCount: 5
electionTimeout: 10000
image: radondb/xenon:v3.0.0
resources:
limits:
cpu: 100m
memory: 256Mi
requests:
cpu: 50m
memory: 128Mi


12 changes: 8 additions & 4 deletions internal/sql_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,13 @@ func (s sqlRunner) QueryRows(query Query) (*sql.Rows, error) {
}

// CheckSlaveStatusWithRetry check the slave status with retry time.
func CheckSlaveStatusWithRetry(sqlRunner SQLRunner, retry uint32) (isLagged, isReplicating corev1.ConditionStatus, err error) {
func CheckSlaveStatusWithRetry(sqlRunner SQLRunner, retry uint32, replicaLag *int32) (isLagged, isReplicating corev1.ConditionStatus, err error) {
for {
if retry == 0 {
break
}

if isLagged, isReplicating, err = CheckSlaveStatus(sqlRunner); err == nil {
if isLagged, isReplicating, err = CheckSlaveStatus(sqlRunner, replicaLag); err == nil {
return
}

Expand All @@ -200,7 +200,7 @@ func CheckSlaveStatusWithRetry(sqlRunner SQLRunner, retry uint32) (isLagged, isR
}

// CheckSlaveStatus check the slave status.
func CheckSlaveStatus(sqlRunner SQLRunner) (isLagged, isReplicating corev1.ConditionStatus, err error) {
func CheckSlaveStatus(sqlRunner SQLRunner, ReplicaLag *int32) (isLagged, isReplicating corev1.ConditionStatus, err error) {
var rows *sql.Rows
isLagged, isReplicating = corev1.ConditionUnknown, corev1.ConditionUnknown
rows, err = sqlRunner.QueryRows(NewQuery("show slave status;"))
Expand Down Expand Up @@ -258,7 +258,11 @@ func CheckSlaveStatus(sqlRunner SQLRunner) (isLagged, isReplicating corev1.Condi

// Check whether the slave is lagged.
sec, _ := strconv.ParseFloat(secondsBehindMaster, 64)
if sec > longQueryTime*100 {
lagTime := longQueryTime * 100
if ReplicaLag != nil {
lagTime = float64(*ReplicaLag)
}
if sec > lagTime {
isLagged = corev1.ConditionTrue
} else {
isLagged = corev1.ConditionFalse
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/syncer/readonly_statefulset.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ func putMySQLReadOnly(s *StatefulSetSyncer, host string) error {
// 3. change master
var isReplicating corev1.ConditionStatus
var err error
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner); err != nil {
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner, s.Spec.ReplicaLag); err != nil {
//Notice!!! this has error, just show error message, can not return.
s.log.V(1).Info("slave status has gotten error", "error", err)
}
Expand Down
4 changes: 2 additions & 2 deletions mysqlcluster/syncer/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ func (s *StatusSyncer) updateNodeStatus(ctx context.Context, cli client.Client,
case <-time.After(time.Second * 5):
}
if sqlRunner != nil {
isLagged, isReplicating, err = internal.CheckSlaveStatusWithRetry(sqlRunner, checkNodeStatusRetry)
isLagged, isReplicating, err = internal.CheckSlaveStatusWithRetry(sqlRunner, checkNodeStatusRetry, s.Spec.ReplicaLag)
if err != nil {
s.log.V(1).Info("failed to check slave status", "node", node.Name, "error", err)
node.Message = err.Error()
Expand Down Expand Up @@ -620,7 +620,7 @@ func (s *StatusSyncer) RoCheckStatus(ctx context.Context, cli client.Client, pod
isCloseSemi = corev1.ConditionTrue
}
// 3. change master
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner); err != nil {
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner, s.Spec.ReplicaLag); err != nil {
node.Message = err.Error()
}
}
Expand Down

0 comments on commit 511a4a0

Please sign in to comment.