Support Cassandra clusters spanning multiple Kubernetes clusters

Instances can be configured to consume external seed node addresses.
d2iq-archive · Apr 30, 2020 · 426b0fd · 426b0fd
1 parent 0cd00b2
commit 426b0fd
Show file tree

Hide file tree

Showing 5 changed files with 141 additions and 1 deletion.
diff --git a/operator/params.yaml b/operator/params.yaml
@@ -205,6 +205,11 @@ parameters:
     description: "This flag can be set to true to automatic installation of a cluster role, service account and role binding"
     default: "false"
 
+  - name: EXTERNAL_SEED_NODES
+    description: "List of seed nodes external to this instance to add to the cluster. This allows clusters spanning multiple Kubernetes clusters."
+    default:
+    type: array
+
   ################################################################################
   ######################### Prometheus exporter settings #########################
   ################################################################################

diff --git a/operator/templates/generate-cassandra-yaml.yaml b/operator/templates/generate-cassandra-yaml.yaml
@@ -492,12 +492,18 @@ data:
                         {{- if or $i $lastTopo -}}, {{- end -}}
                         {{ $.Name }}-{{ $dc.datacenter }}-node-{{ $node }}.{{ $.Name }}-svc.{{ $.Namespace }}.svc.cluster.local
                       {{- end -}}
+                    {{- end -}}
+                    {{- range $external := $.Params.EXTERNAL_SEED_NODES -}}
+                      , {{ . }}
                     {{- end -}}"
           {{- else }}
           # We have a simple setup and use the first 3 nodes as seed
           - seeds: "{{- range $i, $node := until (int (min 3 .Params.NODE_COUNT)) -}}
                       {{- if $i -}}, {{- end -}}
                       {{ $.Name }}-node-{{ $node }}.{{ $.Name }}-svc.{{ $.Namespace }}.svc.cluster.local
+                    {{- end -}}
+                    {{- range $external := $.Params.EXTERNAL_SEED_NODES -}}
+                      , {{ . }}
                     {{- end -}}"
           {{- end }}
 

diff --git a/templates/operator/params.yaml.template b/templates/operator/params.yaml.template
@@ -205,6 +205,11 @@ parameters:
     description: "This flag can be set to true to automatic installation of a cluster role, service account and role binding"
     default: "false"
 
+  - name: EXTERNAL_SEED_NODES
+    description: "List of seed nodes external to this instance to add to the cluster. This allows clusters spanning multiple Kubernetes clusters."
+    default:
+    type: array
+
   ################################################################################
   ######################### Prometheus exporter settings #########################
   ################################################################################

diff --git a/tests/go.mod b/tests/go.mod
@@ -5,7 +5,6 @@ go 1.13
 require (
 	github.com/aws/aws-sdk-go v1.29.14
 	github.com/imdario/mergo v0.3.8 // indirect
-	github.com/kudobuilder/kudo v0.11.1
 	github.com/kudobuilder/test-tools v0.4.0
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1

diff --git a/tests/suites/fault-tolerance/fault_tolerance_test.go b/tests/suites/fault-tolerance/fault_tolerance_test.go
@@ -77,6 +77,25 @@ func buildDatacenterReplicationString(topology cassandra.NodeTopology, maxReplic
 	return result
 }
 
+func getTopology1DatacenterEach1Rack(datacenter, rack string) cassandra.NodeTopology {
+	return cassandra.NodeTopology{
+		{
+			Datacenter: datacenter,
+			DatacenterLabels: map[string]string{
+				nodeSelectorDatacenter: "us-west-2a",
+			},
+			Nodes:        1,
+			RackLabelKey: rackLabelKey,
+			Racks: []cassandra.TopologyRackItem{
+				{
+					Rack:           rack,
+					RackLabelValue: rackLabelValue,
+				},
+			},
+		},
+	}
+}
+
 func getTopology2DatacenterEach1Rack() cassandra.NodeTopology {
 	return cassandra.NodeTopology{
 		{
@@ -311,6 +330,112 @@ var _ = Describe("Fault tolerance tests", func() {
 
 		// TODO: test node selection
 	})
+
+	Context("when having two datacenters in different namespaces", func() {
+		var (
+			dc2Namespace = "fault-tolerance-2"
+		)
+
+		It("is recognized by the respective clusters", func() {
+			var err error
+
+			By("Setting up Namespace and RBAC")
+			err = kubernetes.CreateNamespace(client, testNamespace)
+			if !errors.IsAlreadyExists(err) {
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			err = kubernetes.CreateNamespace(client, dc2Namespace)
+			if !errors.IsAlreadyExists(err) {
+				Expect(err).NotTo(HaveOccurred())
+			}
+
+			defer func() {
+				err := kubernetes.DeleteNamespace(client, dc2Namespace)
+				Expect(err).NotTo(HaveOccurred())
+			}()
+
+			By("Starting the test")
+
+			By("Installing an operator in the first namespace")
+			topology := getTopology1DatacenterEach1Rack("dc1", "rac1")
+			topologyYaml, err := topology.ToYAML()
+			Expect(err).NotTo(HaveOccurred())
+
+			parameters = map[string]string{
+				"NODE_COUNT":                           "1", // NODE_TOPOLOGY should override this value
+				"ENDPOINT_SNITCH":                      "GossipingPropertyFileSnitch",
+				"NODE_TOPOLOGY":                        topologyYaml,
+				"NODE_READINESS_PROBE_INITIAL_DELAY_S": "10",
+				"SERVICE_ACCOUNT_INSTALL":              "true",
+			}
+
+			By("Waiting for the operator to deploy")
+			operator, err = kudo.InstallOperator(operatorDirectory).
+				WithNamespace(testNamespace).
+				WithInstance(instanceName).
+				WithParameters(parameters).
+				Do(client)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = operator.Instance.WaitForPlanComplete("deploy", kudo.WaitTimeout(time.Minute*10))
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Installing an operator in the second namespace with external seed node from the first operator")
+			topology = getTopology1DatacenterEach1Rack("dc2", "rac2")
+			topologyYaml, err = topology.ToYAML()
+			Expect(err).NotTo(HaveOccurred())
+
+			dns := fmt.Sprintf("[%s-dc1-node-0.%s-svc.%s.cluster.local]", instanceName, instanceName, testNamespace)
+
+			parameters = map[string]string{
+				"NODE_COUNT":                           "1", // NODE_TOPOLOGY should override this value
+				"ENDPOINT_SNITCH":                      "GossipingPropertyFileSnitch",
+				"NODE_TOPOLOGY":                        topologyYaml,
+				"NODE_READINESS_PROBE_INITIAL_DELAY_S": "10",
+				"SERVICE_ACCOUNT_INSTALL":              "true",
+				"EXTERNAL_SEED_NODES":                  dns,
+			}
+
+			By("Waiting for the second operator to deploy")
+			operator2, err := kudo.InstallOperator(operatorDirectory).
+				WithNamespace(dc2Namespace).
+				WithInstance(instanceName).
+				WithParameters(parameters).
+				Do(client)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = operator2.Instance.WaitForPlanComplete("deploy", kudo.WaitTimeout(time.Minute*10))
+			Expect(err).NotTo(HaveOccurred())
+
+			nodes, err := cassandra.Nodes(client, operator2.Instance)
+			Expect(err).NotTo(HaveOccurred())
+
+			dcCounts := collectDataCenterCounts(nodes)
+			Expect(dcCounts["dc1"] == 1)
+			Expect(dcCounts["dc2"] == 1)
+
+			By("Updating the external seed nodes of the first operator")
+			dns = fmt.Sprintf("[%s-dc2-node-0.%s-svc.%s.cluster.local]", instanceName, instanceName, dc2Namespace)
+
+			parameters = map[string]string{
+				"EXTERNAL_SEED_NODES": dns,
+			}
+
+			err = operator.Instance.UpdateParameters(parameters)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = operator.Instance.WaitForPlanComplete("deploy", kudo.WaitTimeout(time.Minute*10))
+			Expect(err).NotTo(HaveOccurred())
+
+			nodes, err = cassandra.Nodes(client, operator.Instance)
+			Expect(err).NotTo(HaveOccurred())
+
+			dcCounts = collectDataCenterCounts(nodes)
+			Expect(dcCounts["dc1"] == 1)
+			Expect(dcCounts["dc2"] == 1)
+		})
+	})
 })
 
 func collectDataCenterCounts(nodes []map[string]string) map[string]int {