Improve teardown of system tests to destroy environment if cluster is…

… not in a good state

nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java

@@ -154,8 +154,17 @@ public void teardown() throws Exception {
         logger.info("Beginning teardown");
 
         try {
-            Exception destroyFlowFailure = null;
+            // In some cases a test can pass, but still leave a clustered instance with one of the nodes in a bad state, if the instance then gets reused
+            // it will cause later tests to fail, so it is better to destroy the environment if the cluster is in a bad state at the end of a test
+            final NiFiInstance nifiInstance = nifiRef.get();
+            if (nifiInstance != null && nifiInstance.isClustered() && (!isCoordinatorElected() || !allNodesConnected(nifiInstance.getNumberOfNodes()))) {
+                logger.info("Clustered environment is in a bad state, will completely tear down the environments and start with a clean environment for the next test.");
+                instanceCache.poison(nifiInstance);
+                cleanup();
+                return;
+            }
 
+            Exception destroyFlowFailure = null;
             if (isDestroyFlowAfterEachTest()) {
                 try {
                     destroyFlow();
@@ -598,6 +607,11 @@ protected boolean isCoordinatorElected() throws NiFiClientException, IOException
         return false;
     }
 
+    protected boolean allNodesConnected(int expectedNodeCount) throws NiFiClientException, IOException {
+        final ClusterSummaryEntity clusterSummary = getNifiClient().getFlowClient().getClusterSummary();
+        return expectedNodeCount == clusterSummary.getClusterSummary().getConnectedNodeCount();
+    }
+
     protected void reconnectNode(final int nodeIndex) throws NiFiClientException, IOException {
         final NodeEntity nodeEntity = getNodeEntity(nodeIndex);
         nodeEntity.getNode().setStatus(NodeConnectionState.CONNECTING.name());