From ba81e8e325accc30159f910fee973e86520a7d90 Mon Sep 17 00:00:00 2001
From: Chunyi Lyu <clyu@pivotal.io>
Date: Tue, 26 Jan 2021 17:53:58 +0000
Subject: [PATCH] Set podManagementPolicy to parallel

- use cluster_formation.randomized_startup_delay_range
to avoid race condition between nodes during initial
cluster formation
- this change solves the problem of failing to restart
the cluster when all pods are deleted and recreated at
once; with podManagementPolicy set to 'OrderedReady',
pod 0 is also the first one getting recreated, but it
may not be the last node to shut down.
---
 internal/resource/configmap.go        |  4 +++-
 internal/resource/configmap_test.go   |  2 ++
 internal/resource/statefulset.go      |  1 +
 internal/resource/statefulset_test.go | 11 ++++++++++-
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/internal/resource/configmap.go b/internal/resource/configmap.go
index 09ec839bf..92f405de3 100644
--- a/internal/resource/configmap.go
+++ b/internal/resource/configmap.go
@@ -34,7 +34,9 @@ cluster_formation.k8s.host = kubernetes.default
 cluster_formation.k8s.address_type = hostname
 cluster_partition_handling = pause_minority
 queue_master_locator = min-masters
-disk_free_limit.absolute = 2GB`
+disk_free_limit.absolute = 2GB
+cluster_formation.randomized_startup_delay_range.min = 5
+cluster_formation.randomized_startup_delay_range.max = 30`
 
 	defaultTLSConf = `
 ssl_options.certfile = /etc/rabbitmq-tls/tls.crt
diff --git a/internal/resource/configmap_test.go b/internal/resource/configmap_test.go
index 95ac9ee6b..5c50338a0 100644
--- a/internal/resource/configmap_test.go
+++ b/internal/resource/configmap_test.go
@@ -33,6 +33,8 @@ cluster_formation.k8s.address_type       = hostname
 cluster_partition_handling               = pause_minority
 queue_master_locator                     = min-masters
 disk_free_limit.absolute                 = 2GB
+cluster_formation.randomized_startup_delay_range.min = 5
+cluster_formation.randomized_startup_delay_range.max = 30
 cluster_name                             = ` + instanceName)
 }
 
diff --git a/internal/resource/statefulset.go b/internal/resource/statefulset.go
index 18acbbb73..59122220c 100644
--- a/internal/resource/statefulset.go
+++ b/internal/resource/statefulset.go
@@ -67,6 +67,7 @@ func (builder *StatefulSetBuilder) Build() (client.Object, error) {
 				MatchLabels: metadata.LabelSelector(builder.Instance.Name),
 			},
 			VolumeClaimTemplates: pvc,
+			PodManagementPolicy:  appsv1.ParallelPodManagement,
 		},
 	}
 
diff --git a/internal/resource/statefulset_test.go b/internal/resource/statefulset_test.go
index 3cd302137..cdf9e562a 100644
--- a/internal/resource/statefulset_test.go
+++ b/internal/resource/statefulset_test.go
@@ -65,6 +65,7 @@ var _ = Describe("StatefulSet", func() {
 
 			Expect(statefulSet.Spec.ServiceName).To(Equal(instance.ChildResourceName("nodes")))
 		})
+
 		It("adds the correct label selector", func() {
 			obj, err := stsBuilder.Build()
 			Expect(err).NotTo(HaveOccurred())
@@ -74,7 +75,15 @@ var _ = Describe("StatefulSet", func() {
 			Expect(labels["app.kubernetes.io/name"]).To(Equal(instance.Name))
 		})
 
-		It("references the storageclassname when specified", func() {
+		It("sets pod management policy to 'Parallel' ", func() {
+			obj, err := stsBuilder.Build()
+			Expect(err).NotTo(HaveOccurred())
+			statefulSet := obj.(*appsv1.StatefulSet)
+
+			Expect(statefulSet.Spec.PodManagementPolicy).To(Equal(appsv1.ParallelPodManagement))
+		})
+
+		It("references the storage class name when specified", func() {
 			storageClassName := "my-storage-class"
 			builder.Instance.Spec.Persistence.StorageClassName = &storageClassName