From ffe083847111c5b624c5201679fe7a83f8811fb2 Mon Sep 17 00:00:00 2001
From: acekingke <aceking.ke@gmail.com>
Date: Thu, 7 Apr 2022 10:34:29 +0800
Subject: [PATCH] mysqlcluster: when storage size shrink, and change replicas,
 It will reconcile fail forever. #436

---
 mysqlcluster/syncer/statefulset.go | 20 +++++++++++++++++++-
 sidecar/config.go                  |  2 +-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/mysqlcluster/syncer/statefulset.go b/mysqlcluster/syncer/statefulset.go
index 15bef66a8..be96de776 100644
--- a/mysqlcluster/syncer/statefulset.go
+++ b/mysqlcluster/syncer/statefulset.go
@@ -47,6 +47,9 @@ import (
 // The wait time limit for pod upgrade.
 const waitLimit = 2 * 60 * 60
 
+var startTime time.Time
+var failCount int = 0
+
 // StatefulSetSyncer used to operate statefulset.
 type StatefulSetSyncer struct {
 	*mysqlcluster.MysqlCluster
@@ -146,6 +149,7 @@ func (s *StatefulSetSyncer) Sync(ctx context.Context) (syncer.SyncResult, error)
 		result.SetEventData("Normal", basicEventReason(s.Name, err),
 			fmt.Sprintf("%s %s %s successfully", kind, key, result.Operation))
 		log.Info(string(result.Operation), "key", key, "kind", kind)
+		failCount = 0
 	}
 	return result, err
 }
@@ -167,8 +171,22 @@ func (s *StatefulSetSyncer) canExpandPVC(ctx context.Context) bool {
 	// If newStorage is not greater than oldStorage, do not expand.
 	if newStorage.Cmp(*oldRequest.Storage()) != 1 {
 		log.Info("canExpandPVC", "result", "can not expand", "reason", "new pvc is not larger than old pvc")
-		return false
+		// Should return true, because if change replicas and storage size, it will fail again and again
+		// during expanding PVCs, not great then oldStorage it will not truely expand, just recreate
+		// statefulset.
+		if failCount > 0 && time.Since(startTime) > time.Duration(30)*time.Second {
+			log.Info("canExpandPVC", "result", "reason", "over 30 seconds, trying to expand")
+			failCount = 0
+			return true
+		} else {
+			if failCount == 0 {
+				startTime = time.Now()
+			}
+			failCount++
+			return false
+		}
 	}
+	failCount = 0
 	return true
 }
 
diff --git a/sidecar/config.go b/sidecar/config.go
index eeaa5552d..57c99c0dd 100644
--- a/sidecar/config.go
+++ b/sidecar/config.go
@@ -333,7 +333,7 @@ func (cfg *Config) buildXenonConf() []byte {
 			"admin": "root",
 			"ping-timeout": %d,
 			"passwd": "%s",
-			"host": "localhost",
+			"host": "127.0.0.1",
 			"version": "%s",
 			"master-sysvars": "%s",
 			"slave-sysvars": "%s",