Implement Readiness Probe

operator/operator.yaml

@@ -16,6 +16,7 @@ tasks:
       - cassandra-env-sh.yaml
       - jvm-options.yaml
       - stateful-set.yaml
+      - node-readiness-probe-sh.yaml
 
 plans:
   deploy:

operator/params.yaml

@@ -30,6 +30,26 @@ NODE_STORAGE_CLASS:
   description: "The storage class to be used in volumeClaimTemplates. By default, it is not required and the default storage class is used."
   required: false
 
+NODE_READINESS_PROBE_INITIAL_DELAY_S:
+  description: "Number of seconds after the container has started before the readiness probe is initiated."
+  default: "0"
+
+NODE_READINESS_PROBE_PERIOD_S:
+  description: "How often (in seconds) to perform the readiness probe."
+  default: "5"
+
+NODE_READINESS_PROBE_TIMEOUT_S:
+  description: "How long (in seconds) to wait for a readiness probe to succeed."
+  default: "60"
+
+NODE_READINESS_PROBE_SUCCESS_THRESHOLD:
+  description: "Minimum consecutive successes for the probe to be considered successful after having failed."
+  default: "1"
+
+NODE_READINESS_PROBE_FAILURE_THRESHOLD:
+  description: "When a pod starts and the probe fails, `failure_threshold` attempts will be made before marking the pod as 'unready'."
+  default: "3"
+
 OVERRIDE_CLUSTER_NAME:
   description: "Override the name of the Cassandra cluster set by the operator. This shouldn't be explicit set, unless you know what you're doing."
   default: ""

operator/templates/node-readiness-probe-sh.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-readiness-probe-sh
+data:
+  node-readiness-probe.sh: |
+    nodetool status -p {{ .Params.JMX_PORT }} | grep -q "UN  ${POD_IP}"

operator/templates/stateful-set.yaml

@@ -62,6 +62,16 @@ spec:
                 command:
                   - nodetool
                   - drain
+          readinessProbe:
+            exec:
+              command:
+                  - /bin/bash
+                  - /etc/cassandra/node-readiness-probe.sh
+            initialDelaySeconds: {{ .Params.NODE_READINESS_PROBE_INITIAL_DELAY_S }}
+            periodSeconds: {{ .Params.NODE_READINESS_PROBE_PERIOD_S }}
+            timeoutSeconds: {{ .Params.NODE_READINESS_PROBE_TIMEOUT_S }}
+            successThreshold: {{ .Params.NODE_READINESS_PROBE_SUCCESS_THRESHOLD }}
+            failureThreshold: {{ .Params.NODE_READINESS_PROBE_FAILURE_THRESHOLD }}
           command:
             - cassandra
             - -f
@@ -133,6 +143,9 @@ spec:
             - name: jvm-options
               mountPath: /etc/cassandra/jvm.options
               subPath: jvm.options
+            - name: node-readiness-probe-sh
+              mountPath: /etc/cassandra/node-readiness-probe.sh
+              subPath: node-readiness-probe.sh
       volumes:
         # Overwriting /etc/cassandra/ available in the Docker image.
         - name: etc-cassandra
@@ -146,6 +159,9 @@ spec:
         - name: jvm-options
           configMap:
             name: {{ .Name }}-jvm-options
+        - name: node-readiness-probe-sh
+          configMap:
+            name: {{ .Name }}-node-readiness-probe-sh
   volumeClaimTemplates:
     - metadata:
         name: var-lib-cassandra

templates/operator/operator.yaml.template

@@ -16,6 +16,7 @@ tasks:
       - cassandra-env-sh.yaml
       - jvm-options.yaml
       - stateful-set.yaml
+      - node-readiness-probe-sh.yaml
 
 plans:
   deploy: