Stop scheduled Pod kill for operators during restart all test

FoundationDB · Feb 23, 2024 · 91157f5 · 91157f5
1 parent 25ed035
commit 91157f5
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 11 deletions.
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
@@ -137,7 +137,7 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		clusterLog.Info("Fetch machine-readable status for reconcilitation loop", "cacheStatus", cacheStatus)
 		status, err = r.getStatusFromClusterOrDummyStatus(clusterLog, cluster)
 		if err != nil {
-			clusterLog.Info("could not fetch machine-readable and therefor don't cache the status")
+			clusterLog.Info("could not fetch machine-readable status and therefore didn't cache the it")
 		}
 	}
 

diff --git a/e2e/test_operator/operator_test.go b/e2e/test_operator/operator_test.go
@@ -49,9 +49,10 @@ import (
 )
 
 var (
-	factory     *fixtures.Factory
-	fdbCluster  *fixtures.FdbCluster
-	testOptions *fixtures.FactoryOptions
+	factory               *fixtures.Factory
+	fdbCluster            *fixtures.FdbCluster
+	testOptions           *fixtures.FactoryOptions
+	scheduleInjectPodKill *fixtures.ChaosMeshExperiment
 )
 
 func init() {
@@ -81,7 +82,7 @@ var _ = BeforeSuite(func() {
 
 	// In order to test the robustness of the operator we try to kill the operator Pods every minute.
 	if factory.ChaosTestsEnabled() {
-		factory.ScheduleInjectPodKill(
+		scheduleInjectPodKill = factory.ScheduleInjectPodKill(
 			fixtures.GetOperatorSelector(fdbCluster.Namespace()),
 			"*/2 * * * *",
 			chaosmesh.OneMode,
@@ -1898,6 +1899,8 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 		var initialSetting bool
 
 		BeforeEach(func() {
+			// Until the race condition is resolved in the FDB go bindings make sure the operator is not restarted.
+			factory.DeleteChaosMeshExperimentSafe(scheduleInjectPodKill)
 			cluster := fdbCluster.GetCluster()
 			parsedVersion, err := fdbv1beta2.ParseFdbVersion(cluster.Status.RunningVersion)
 			Expect(err).NotTo(HaveOccurred())
@@ -1914,6 +1917,14 @@ var _ = Describe("Operator", Label("e2e", "pr"), func() {
 
 		AfterEach(func() {
 			Expect(fdbCluster.SetUseDNSInClusterFile(initialSetting)).ToNot(HaveOccurred())
+
+			if factory.ChaosTestsEnabled() {
+				scheduleInjectPodKill = factory.ScheduleInjectPodKill(
+					fixtures.GetOperatorSelector(fdbCluster.Namespace()),
+					"*/2 * * * *",
+					chaosmesh.OneMode,
+				)
+			}
 		})
 
 		When("all Pods are deleted", func() {

diff --git a/go.mod b/go.mod
@@ -30,10 +30,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
-// Points to https://github.com/apple/foundationdb/pull/11221 once those changes are merged, update the import above
-// and remove the replace statement.
-replace github.com/apple/foundationdb/bindings/go => github.com/johscheuer/foundationdb/bindings/go v0.0.0-20240223074533-f06c55dce3a5
-
 require golang.org/x/sync v0.3.0
 
 require (

diff --git a/go.sum b/go.sum
@@ -52,6 +52,8 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/apple/foundationdb/bindings/go v0.0.0-20231020161252-ed27c828ca16 h1:+FSWyyQT1jj2LHHRsIWtTqWcJRmKSJP3IYjOb0NMutU=
+github.com/apple/foundationdb/bindings/go v0.0.0-20231020161252-ed27c828ca16/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
 github.com/apple/foundationdb/fdbkubernetesmonitor v0.0.0-20220513200452-e6fa4d7422d2 h1:qQW+EDheBFF09sjMwEJu7cc6LBQKnsbIVTgj9i12lws=
 github.com/apple/foundationdb/fdbkubernetesmonitor v0.0.0-20220513200452-e6fa4d7422d2/go.mod h1:LgBm9afX7nbQnDQa6bOXluKRnXypEDni36EJExtih80=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -244,8 +246,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/johscheuer/foundationdb/bindings/go v0.0.0-20240223074533-f06c55dce3a5 h1:w04jvvgNZTGeECE3Onf4a9pc3ilxpjWIORqN/AYdAbI=
-github.com/johscheuer/foundationdb/bindings/go v0.0.0-20240223074533-f06c55dce3a5/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=