Merge pull request #361 from olliewalsh/tls_change

Force recreation of ovs db cluster on first replica when protocol changes

kuttl-test.yaml

@@ -19,7 +19,7 @@ kind: TestSuite
 reportFormat: JSON
 reportName: kuttl-test-ovn
 namespace: openstack
-timeout: 360
+timeout: 600
 parallel: 1
 suppress:
   - events                     # Remove spammy event logs

pkg/ovndbcluster/statefulset.go

@@ -54,6 +54,13 @@ func StatefulSet(
 		PeriodSeconds:       5,
 		InitialDelaySeconds: 5,
 	}
+	startupProbe := &corev1.Probe{
+		// TODO might need tuning
+		TimeoutSeconds:      5,
+		PeriodSeconds:       3,
+		FailureThreshold:    20,
+		InitialDelaySeconds: 3,
+	}
 
 	var preStopCmd []string
 	cmd := []string{"/usr/bin/dumb-init"}
@@ -67,6 +74,7 @@ func StatefulSet(
 		},
 	}
 	readinessProbe.Exec = livenessProbe.Exec
+	startupProbe.Exec = livenessProbe.Exec
 
 	preStopCmd = []string{
 		"/usr/local/bin/container-scripts/cleanup.sh",
@@ -159,6 +167,7 @@ func StatefulSet(
 							Resources:                instance.Spec.Resources,
 							ReadinessProbe:           readinessProbe,
 							LivenessProbe:            livenessProbe,
+							StartupProbe:             startupProbe,
 							Lifecycle:                lifecycle,
 							TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
 						},

templates/ovndbcluster/bin/setup.sh

@@ -87,6 +87,26 @@ if ! [ -s $DB_FILE ]; then
     cleanup_db_file
 fi
 
+# Must remove a cluster member to change protocol, replicas 1/2 will have
+# left the cluster when terminating the pod, but cannot remove final member from
+# a cluster (replica 0).
+# Convert db to standalone mode on this member instead.
+# Cluster then gets recreated by ovnctl run_*b_ovsdb using the new local address.
+if [ "$(hostname)" == "{{ .SERVICE_NAME }}-0" ]; then
+    DB_LOCAL_ADDR={{ if .TLS }}ssl{{ else }}tcp{{ end }}:$(hostname).{{ .SERVICE_NAME }}.${NAMESPACE}.svc.cluster.local:${RAFT_PORT}
+    if [ -e ${DB_FILE} ] && \
+       ovsdb-tool db-is-clustered ${DB_FILE} && \
+       ACTUAL_DB_LOCAL_ADDR="$(ovsdb-tool db-local-address ${DB_FILE})" && \
+       [ "${ACTUAL_DB_LOCAL_ADDR}" != "${DB_LOCAL_ADDR}" ] \
+       ; \
+    then
+        rm -f "${DB_FILE%.db}_standalone.db"
+        if ovsdb-tool cluster-to-standalone "${DB_FILE%.db}_standalone.db" "${DB_FILE}"; then
+            mv -f "${DB_FILE%.db}_standalone.db" "${DB_FILE}"
+        fi
+    fi
+fi
+
 # Wait until the ovsdb-tool finishes.
 trap wait_for_ovsdb_tool EXIT
 
@@ -106,8 +126,10 @@ if [[ "$(hostname)" == "{{ .SERVICE_NAME }}-0" ]]; then
 
 {{- if .TLS }}
     ${CTLCMD} set-ssl {{.OVNDB_KEY_PATH}} {{.OVNDB_CERT_PATH}} {{.OVNDB_CACERT_PATH}}
-    ${CTLCMD} set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR}
+{{- else }}
+    ${CTLCMD} del-ssl
 {{- end }}
+    ${CTLCMD} set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR}
 
     # OVN does not support setting inactivity-probe through --remote cli arg so
     # we have to set it after database is up.

tests/kuttl/common/errors_cleanup_ovn.yaml

@@ -65,14 +65,12 @@ kind: Pod
 metadata:
   labels:
     service: ovsdbserver-nb
-  name: ovsdbserver-nb-0
 ---
 apiVersion: v1
 kind: Pod
 metadata:
   labels:
     service: ovsdbserver-sb
-  name: ovsdbserver-sb-0
 ---
 apiVersion: v1
 kind: Pod
@@ -97,27 +95,9 @@ kind: Service
 metadata:
   labels:
     service: ovsdbserver-nb
-  name: ovsdbserver-nb
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    service: ovsdbserver-nb
-    statefulset.kubernetes.io/pod-name: ovsdbserver-nb-0
-  name: ovsdbserver-nb-0
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    service: ovsdbserver-sb
-  name: ovsdbserver-sb
 ---
 apiVersion: v1
 kind: Service
 metadata:
   labels:
     service: ovsdbserver-sb
-    statefulset.kubernetes.io/pod-name: ovsdbserver-sb-0
-  name: ovsdbserver-sb-0

tests/kuttl/common/scripts/check_cluster_status.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
-# arguments: db-type: {nb, sb}, num-pods
+# arguments: db-type: {nb, sb}, num-pods, ssl/tcp
 # Check arguments
 if [ $# -lt 2 ]; then
-    echo "Usage: $0 <db-type> <num-pods>"
+    echo "Usage: $0 <db-type> <num-pods> [ssl]"
     exit 1
 fi
 
 DB_TYPE="$1"
 NUM_PODS="$2"
+PROTOCOL="${3:-tcp}"
 POD_PREFIX="ovsdbserver-${DB_TYPE}"
 CTL_FILE="ovn${DB_TYPE}_db.ctl"
 if [ "$DB_TYPE" == "nb" ]; then
@@ -46,7 +47,7 @@ for pod in "${pods[@]}"; do
     # check if the pod is connected with all other pods
     for server in "${pods[@]}"; do
     echo "Checking if $server is mentioned in the output"
-    if ! echo "$output" | grep -q "$server"; then
+    if ! echo "$output" | grep -q "$PROTOCOL:$server"; then
         exit 1
     fi
     done

tests/kuttl/tests/ovn_db_delete/03-assert.yaml

@@ -3,7 +3,5 @@ kind: TestAssert
 commands:
   - script: |
       $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 3
-      test $? -eq 0
   - script: |
       $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 3
-      test $? -eq 0

tests/kuttl/tests/ovn_db_delete/09-assert.yaml

@@ -3,7 +3,5 @@ kind: TestAssert
 commands:
   - script: |
       $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 1
-      test $? -eq 0
   - script: |
       $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 1
-      test $? -eq 0

tests/kuttl/tests/ovn_tls_enable/00-assert.yaml

@@ -0,0 +1 @@
+../../common/assert_tls_certs.yaml

tests/kuttl/tests/ovn_tls_enable/00-tls_ca_bundle.yaml

@@ -0,0 +1 @@
+../../common/tls_ca_bundle.yaml

tests/kuttl/tests/ovn_tls_enable/00-tls_certs.yaml

@@ -0,0 +1 @@
+../../common/tls_certs.yaml

tests/kuttl/tests/ovn_tls_enable/01-assert.yaml

@@ -0,0 +1,39 @@
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  name: ovndbcluster-nb-sample
+spec:
+  replicas: 3
+status:
+  readyCount: 3
+---
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  name: ovndbcluster-sb-sample
+spec:
+  replicas: 3
+status:
+  readyCount: 3
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  # check the DB uri scheme is tcp
+  - script: |
+      template='{{.status.internalDbAddress}}{{"\n"}}'
+      regex="tcp:.*"
+      dbUri=$(oc get -n $NAMESPACE OVNDBCluster ovndbcluster-sb-sample -o go-template="$template")
+      matches=$(echo "$dbUri" | sed -e "s?$regex??")
+      if [[ -n "$matches" ]]; then
+        exit 1
+      fi
+  # Check ovn connect is ptcp
+  - script: |
+      sb_pod=$(oc get pod -n $NAMESPACE -l service=ovsdbserver-sb -o name|head -1)
+      oc rsh -n $NAMESPACE ${sb_pod} ovn-sbctl --no-leader-only  get-connection | grep -q ptcp
+  # Check we have 3 servers using tcp
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 3
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 3

tests/kuttl/tests/ovn_tls_enable/01-deploy-ovn.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      cp ../../../../config/samples/ovn_* deploy/
+      oc kustomize deploy | oc apply -n $NAMESPACE -f -

tests/kuttl/tests/ovn_tls_enable/02-assert.yaml

@@ -0,0 +1,39 @@
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  name: ovndbcluster-nb-sample
+spec:
+  replicas: 3
+status:
+  readyCount: 3
+---
+apiVersion: ovn.openstack.org/v1beta1
+kind: OVNDBCluster
+metadata:
+  name: ovndbcluster-sb-sample
+spec:
+  replicas: 3
+status:
+  readyCount: 3
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  # check the DB uri scheme is ssl
+  - script: |
+      template='{{.status.internalDbAddress}}{{"\n"}}'
+      regex="ssl:.*"
+      dbUri=$(oc get -n $NAMESPACE OVNDBCluster ovndbcluster-sb-sample -o go-template="$template")
+      matches=$(echo "$dbUri" | sed -e "s?$regex??")
+      if [[ -n "$matches" ]]; then
+        exit 1
+      fi
+  # Check ovn connect is pssl
+  - script: |
+      sb_pod=$(oc get pod -n $NAMESPACE -l service=ovsdbserver-sb -o name|head -1)
+      oc rsh -n $NAMESPACE ${sb_pod} ovn-sbctl --no-leader-only  get-connection | grep -q pssl
+  # Check we have 3 servers using ssl
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 3 ssl
+  - script: |
+      $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 3 ssl

tests/kuttl/tests/ovn_tls_enable/02-enable-tls.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      cp ../../../../config/samples/ovn_* enable_tls/
+      oc kustomize enable_tls | oc apply -n $NAMESPACE -f -

tests/kuttl/tests/ovn_tls_enable/03-cleanup.yaml

@@ -0,0 +1 @@
+../../common/cleanup-ovn.yaml

tests/kuttl/tests/ovn_tls_enable/03-errors.yaml

@@ -0,0 +1 @@
+../../common/errors_cleanup_ovn.yaml

tests/kuttl/tests/ovn_tls_enable/deploy/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ovn_v1beta1_ovndbcluster.yaml
+patches:
+- patch: |-
+    - op: add
+      path: /spec/replicas
+      value: 3
+  target:
+    kind: OVNDBCluster

tests/kuttl/tests/ovn_tls_enable/enable_tls/kustomization.yaml

@@ -0,0 +1,47 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ovn_v1beta1_ovnnorthd.yaml
+- ovn_v1beta1_ovndbcluster.yaml
+- ovn_v1beta1_ovncontroller.yaml
+patches:
+- patch: |-
+    - op: add
+      path: /spec/replicas
+      value: 3
+  target:
+    kind: OVNDBCluster
+- patch: |-
+    - op: add
+      path: /spec/tls
+      value:
+        caBundleSecretName: combined-ca-bundle
+        secretName: cert-ovsdbserver-nb-svc
+  target:
+    kind: OVNDBCluster
+    name: ovndbcluster-nb-sample
+- patch: |-
+    - op: add
+      path: /spec/tls
+      value:
+        caBundleSecretName: combined-ca-bundle
+        secretName: cert-ovsdbserver-sb-svc
+  target:
+    kind: OVNDBCluster
+    name: ovndbcluster-sb-sample
+- patch: |-
+    - op: add
+      path: /spec/tls
+      value:
+        caBundleSecretName: combined-ca-bundle
+        secretName: cert-ovnnorthd-svc
+  target:
+    kind: OVNNorthd
+- patch: |-
+    - op: add
+      path: /spec/tls
+      value:
+        caBundleSecretName: combined-ca-bundle
+        secretName: cert-ovncontroller-svc
+  target:
+    kind: OVNController