Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: add replica lag parameter #812

Merged
merged 1 commit into from
Aug 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/v1alpha1/mysqlcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ type MysqlClusterSpec struct {
// Readonlys Info.
// +optional
ReadOnlys *ReadOnlyType `json:"readonlys,omitempty"`
// Lagged
ReplicaLag *int32 `json:"lag,omitempty"`
// The number of pods from that set that must still be available after the
// eviction, even in the absence of the evicted pod
// +optional
Expand Down
5 changes: 5 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions api/v1beta1/mysqlcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ type MysqlClusterSpec struct {
// Readonlys Info.
// +optional
ReadOnlys *ReadOnlyType `json:"readonlys,omitempty"`
// Lagged
ReplicaLag *int32 `json:"lag,omitempty"`

// Username of new user to create.
// Only be a combination of letters, numbers or underlines. The length can not exceed 26 characters.
// +optional
Expand Down
2 changes: 2 additions & 0 deletions api/v1beta1/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ spec:
s3Schedule:
type: string
type: object
lag:
description: Lagged
format: int32
type: integer
metricsOpts:
default:
enabled: false
Expand Down Expand Up @@ -3526,6 +3530,10 @@ spec:
- Never
- IfNotPresent
type: string
lag:
description: Lagged
format: int32
type: integer
logOpts:
description: LogOpts is the options of log settings.
properties:
Expand Down
8 changes: 8 additions & 0 deletions config/crd/bases/mysql.radondb.com_mysqlclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ spec:
s3Schedule:
type: string
type: object
lag:
description: Lagged
format: int32
type: integer
metricsOpts:
default:
enabled: false
Expand Down Expand Up @@ -3508,6 +3512,10 @@ spec:
- Never
- IfNotPresent
type: string
lag:
description: Lagged
format: int32
type: integer
logOpts:
description: LogOpts is the options of log settings.
properties:
Expand Down
13 changes: 8 additions & 5 deletions config/samples/mysql_v1beta1_backup.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ kind: Backup
metadata:
name: backup-sample
spec:
# Add fields here
image: radondb/mysql57-sidecar:v3.0.0
# hostfrom if empty, use the leader as hostfrom
hostfrom: sample-mysql-0
backupops:
s3:
secretName: sample-backup-secret
clusterName: sample
# nfsServerAddress: ""
method: xtrabackup
# schedule:
# cronExpression: "*/2 * * * *"
# type: s3

72 changes: 64 additions & 8 deletions config/samples/mysql_v1beta1_mysqlcluster.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,68 @@
apiVersion: mysql.radondb.com/v1beta1
kind: MysqlCluster
metadata:
labels:
app.kubernetes.io/name: mysqlcluster
app.kubernetes.io/instance: mysqlcluster-sample
app.kubernetes.io/part-of: radondb-mysql-kubernetes
app.kubernetes.io/managed-by: kustomize
app.kubernetes.io/created-by: radondb-mysql-kubernetes
name: mysqlcluster-sample
name: sample
spec:
# TODO(user): Add fields here
backupOpts:
image: radondb/mysql80-sidecar:v3.0.0
resources:
requests:
cpu: 10m
memory: 32Mi
customTLSSecret: {}
dataSource:
S3backup:
name: ""
secretName: ""
remote: {}
image: percona/percona-server:8.0.25
imagePullPolicy: Always
logOpts:
resources:
requests:
cpu: 10m
memory: 32Mi
maxLagTime: 30
minAvailable: 50%
monitoringSpec:
exporter:
enabled: true
image: prom/mysqld-exporter:v0.12.1
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 10m
memory: 32Mi
mysqlConfig: {}
mysqlVersion: "8.0"
replicas: 3
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 100m
memory: 256Mi
storage:
accessModes:
- ReadWriteOnce
storageClassName: local-path
resources:
requests:
storage: 20Gi
user: radondb_usr
xenonOpts:
admitDefeatHearbeatCount: 5
electionTimeout: 10000
image: radondb/xenon:v3.0.0
resources:
limits:
cpu: 100m
memory: 256Mi
requests:
cpu: 50m
memory: 128Mi


12 changes: 8 additions & 4 deletions internal/sql_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,13 @@ func (s sqlRunner) QueryRows(query Query) (*sql.Rows, error) {
}

// CheckSlaveStatusWithRetry check the slave status with retry time.
func CheckSlaveStatusWithRetry(sqlRunner SQLRunner, retry uint32) (isLagged, isReplicating corev1.ConditionStatus, err error) {
func CheckSlaveStatusWithRetry(sqlRunner SQLRunner, retry uint32, replicaLag *int32) (isLagged, isReplicating corev1.ConditionStatus, err error) {
for {
if retry == 0 {
break
}

if isLagged, isReplicating, err = CheckSlaveStatus(sqlRunner); err == nil {
if isLagged, isReplicating, err = CheckSlaveStatus(sqlRunner, replicaLag); err == nil {
return
}

Expand All @@ -200,7 +200,7 @@ func CheckSlaveStatusWithRetry(sqlRunner SQLRunner, retry uint32) (isLagged, isR
}

// CheckSlaveStatus check the slave status.
func CheckSlaveStatus(sqlRunner SQLRunner) (isLagged, isReplicating corev1.ConditionStatus, err error) {
func CheckSlaveStatus(sqlRunner SQLRunner, ReplicaLag *int32) (isLagged, isReplicating corev1.ConditionStatus, err error) {
var rows *sql.Rows
isLagged, isReplicating = corev1.ConditionUnknown, corev1.ConditionUnknown
rows, err = sqlRunner.QueryRows(NewQuery("show slave status;"))
Expand Down Expand Up @@ -258,7 +258,11 @@ func CheckSlaveStatus(sqlRunner SQLRunner) (isLagged, isReplicating corev1.Condi

// Check whether the slave is lagged.
sec, _ := strconv.ParseFloat(secondsBehindMaster, 64)
if sec > longQueryTime*100 {
lagTime := longQueryTime * 100
if ReplicaLag != nil {
lagTime = float64(*ReplicaLag)
}
if sec > lagTime {
isLagged = corev1.ConditionTrue
} else {
isLagged = corev1.ConditionFalse
Expand Down
2 changes: 1 addition & 1 deletion mysqlcluster/syncer/readonly_statefulset.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ func putMySQLReadOnly(s *StatefulSetSyncer, host string) error {
// 3. change master
var isReplicating corev1.ConditionStatus
var err error
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner); err != nil {
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner, s.Spec.ReplicaLag); err != nil {
//Notice!!! this has error, just show error message, can not return.
s.log.V(1).Info("slave status has gotten error", "error", err)
}
Expand Down
4 changes: 2 additions & 2 deletions mysqlcluster/syncer/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ func (s *StatusSyncer) updateNodeStatus(ctx context.Context, cli client.Client,
case <-time.After(time.Second * 5):
}
if sqlRunner != nil {
isLagged, isReplicating, err = internal.CheckSlaveStatusWithRetry(sqlRunner, checkNodeStatusRetry)
isLagged, isReplicating, err = internal.CheckSlaveStatusWithRetry(sqlRunner, checkNodeStatusRetry, s.Spec.ReplicaLag)
if err != nil {
s.log.V(1).Info("failed to check slave status", "node", node.Name, "error", err)
node.Message = err.Error()
Expand Down Expand Up @@ -616,7 +616,7 @@ func (s *StatusSyncer) RoCheckStatus(ctx context.Context, cli client.Client, pod
isCloseSemi = corev1.ConditionTrue
}
// 3. change master
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner); err != nil {
if _, isReplicating, err = internal.CheckSlaveStatus(sqlRunner, s.Spec.ReplicaLag); err != nil {
node.Message = err.Error()
}
}
Expand Down