[PAN-1625] Clique AT mining continues if validator offline

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/clique/CliqueMiningAcceptanceTest.java

@@ -17,6 +17,7 @@
 import tech.pegasys.pantheon.tests.acceptance.dsl.node.PantheonNode;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.junit.Test;
 
@@ -72,4 +73,31 @@ public void shouldStallMiningWhenInsufficientValidators() throws IOException {
     minerNode1.verify(net.awaitPeerCount(0));
     minerNode1.verify(clique.noNewBlockCreated(minerNode1));
   }
+
+  @Test
+  public void shouldStillMineWhenANonProposerNodeFailsAndHasSufficientValidators()
+      throws IOException {
+    final PantheonNode[] nodes = {
+      pantheon.createCliqueNode("miner1"),
+      pantheon.createCliqueNode("miner2"),
+      pantheon.createCliqueNode("miner3")
+    };
+    final List<PantheonNode> validators = clique.validators(nodes);
+    final PantheonNode validator1 = validators.get(0);
+    final PantheonNode validator2 = validators.get(1);
+    final PantheonNode validator3 = validators.get(2);
+
+    cluster.start(validators);
+    cluster.waitUntil(wait.chainHeadHasProgressedByAtLeast(validator1, 1, 85));
+
+    cluster.stopNode(validator3);
+    validator1.verify(net.awaitPeerCount(1));
+    validator2.verify(net.awaitPeerCount(1));
+
+    cluster.waitOnActiveNodesUntil(wait.chainHeadHasProgressedByAtLeast(validator1, 1));
+    cluster.verifyOnActiveNodes(clique.proposerAtChainHeadEqual(validator1));
+
+    cluster.waitOnActiveNodesUntil(wait.chainHeadHasProgressedByAtLeast(validator1, 1));
+    cluster.verifyOnActiveNodes(clique.proposerAtChainHeadEqual(validator2));
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/BlockUtils.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.tests.acceptance.dsl;
+
+import static tech.pegasys.pantheon.ethereum.core.Hash.fromHexString;
+
+import tech.pegasys.pantheon.ethereum.core.Address;
+import tech.pegasys.pantheon.ethereum.core.BlockHeader;
+import tech.pegasys.pantheon.ethereum.core.BlockHeaderFunctions;
+import tech.pegasys.pantheon.ethereum.core.Hash;
+import tech.pegasys.pantheon.ethereum.core.LogsBloomFilter;
+import tech.pegasys.pantheon.util.bytes.BytesValue;
+import tech.pegasys.pantheon.util.uint.UInt256;
+
+import org.web3j.protocol.core.methods.response.EthBlock.Block;
+
+public class BlockUtils {
+
+  public static BlockHeader createBlockHeader(
+      final Block block, final BlockHeaderFunctions blockHeaderFunctions) {
+    final Hash mixHash =
+        block.getMixHash() == null
+            ? Hash.fromHexStringLenient("0x0")
+            : fromHexString(block.getMixHash());
+    return new BlockHeader(
+        fromHexString(block.getParentHash()),
+        fromHexString(block.getSha3Uncles()),
+        Address.fromHexString(block.getMiner()),
+        fromHexString(block.getStateRoot()),
+        fromHexString(block.getTransactionsRoot()),
+        fromHexString(block.getReceiptsRoot()),
+        LogsBloomFilter.fromHexString(block.getLogsBloom()),
+        UInt256.fromHexString(block.getDifficultyRaw()),
+        block.getNumber().longValue(),
+        block.getGasLimit().longValue(),
+        block.getGasUsed().longValue(),
+        block.getTimestamp().longValue(),
+        BytesValue.fromHexString(block.getExtraData()),
+        mixHash,
+        block.getNonce().longValue(),
+        blockHeaderFunctions);
+  }
+}

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/condition/clique/ExpectBlockHasProposer.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2018 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.tests.acceptance.dsl.condition.clique;
+
+import static org.assertj.core.api.Java6Assertions.assertThat;
+import static tech.pegasys.pantheon.tests.acceptance.dsl.BlockUtils.createBlockHeader;
+import static tech.pegasys.pantheon.tests.acceptance.dsl.WaitUtils.waitFor;
+
+import tech.pegasys.pantheon.consensus.clique.CliqueBlockHeaderFunctions;
+import tech.pegasys.pantheon.consensus.clique.CliqueExtraData;
+import tech.pegasys.pantheon.ethereum.core.Address;
+import tech.pegasys.pantheon.ethereum.core.BlockHeader;
+import tech.pegasys.pantheon.tests.acceptance.dsl.condition.Condition;
+import tech.pegasys.pantheon.tests.acceptance.dsl.node.Node;
+import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.eth.EthTransactions;
+
+import org.web3j.protocol.core.methods.response.EthBlock.Block;
+
+public class ExpectBlockHasProposer implements Condition {
+  private final EthTransactions eth;
+  private Address proposer;
+
+  public ExpectBlockHasProposer(final EthTransactions eth, final Address proposer) {
+    this.eth = eth;
+    this.proposer = proposer;
+  }
+
+  @Override
+  public void verify(final Node node) {
+    waitFor(() -> assertThat(proposerAddress(node)).isEqualTo(proposer));
+  }
+
+  private Address proposerAddress(final Node node) {
+    final Block block = node.execute(eth.block());
+    final BlockHeader blockHeader = createBlockHeader(block, new CliqueBlockHeaderFunctions());
+    final CliqueExtraData cliqueExtraData = CliqueExtraData.decode(blockHeader);
+    return cliqueExtraData.getProposerAddress();
+  }
+}

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/jsonrpc/Clique.java

@@ -20,6 +20,7 @@
 import tech.pegasys.pantheon.ethereum.core.Address;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.Condition;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.blockchain.ExpectBlockNotCreated;
+import tech.pegasys.pantheon.tests.acceptance.dsl.condition.clique.ExpectBlockHasProposer;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.clique.ExpectNonceVote;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.clique.ExpectNonceVote.CLIQUE_NONCE_VOTE;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.clique.ExpectProposals;
@@ -33,7 +34,9 @@
 
 import java.math.BigInteger;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.stream.Collectors;
@@ -50,6 +53,14 @@ public Clique(final EthTransactions eth, final CliqueTransactions clique) {
     this.clique = clique;
   }
 
+  public List<PantheonNode> validators(final PantheonNode[] nodes) {
+    final Comparator<PantheonNode> compareByAddress =
+        Comparator.comparing(PantheonNode::getAddress);
+    List<PantheonNode> pantheonNodes = Arrays.asList(nodes);
+    pantheonNodes.sort(compareByAddress);
+    return pantheonNodes;
+  }
+
   public ExpectValidators validatorsEqual(final PantheonNode... validators) {
     return new ExpectValidators(clique, validatorAddresses(validators));
   }
@@ -98,6 +109,10 @@ private Address[] validatorAddresses(final PantheonNode[] validators) {
     return Arrays.stream(validators).map(PantheonNode::getAddress).sorted().toArray(Address[]::new);
   }
 
+  public Condition proposerAtChainHeadEqual(final PantheonNode proposer) {
+    return new ExpectBlockHasProposer(eth, proposer.getAddress());
+  }
+
   public static class ProposalsConfig {
     private final Map<PantheonNode, Boolean> proposals = new HashMap<>();
     private final CliqueTransactions clique;

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/cluster/Cluster.java

@@ -174,4 +174,10 @@ public void waitUntil(final WaitCondition condition) {
       node.waitUntil(condition);
     }
   }
+
+  public void waitOnActiveNodesUntil(final WaitCondition condition) {
+    nodes.values().stream()
+        .filter(node -> pantheonNodeRunner.isActive(node.getName()))
+        .forEach(condition::waitUntil);
+  }
 }