Merge pull request #346 from HorizenOfficial/dev - Release 0.3.1

Release 0.3.1

CHANGELOG.md

@@ -1,3 +1,7 @@
+**0.3.1**
+1. Withdrawal epoch validator: fix wrongly rejected sidechain block containing McBlockRef with MC2SCAggTx leading to the end of the withdrawal epoch.
+
+
 **Blaze changes (0.3.0)**
 1. New proving system for certificates verification: Coboundary Marlin.
 2. PGD: decentralized certificates signing.

README.md

@@ -40,7 +40,7 @@ While we keep monitoring the memory footprint of the proofs generation process,
  - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows:
 
 ```
-LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.0.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.1.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
 ```
  - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. 
 

ci/run_sc.sh

@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.0}"
+SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.1}"
 
 if [ -d "$1" ] && [ -f "$2" ]; then
   path_to_jemalloc="$(ldconfig -p | grep "$(arch)" | grep 'libjemalloc\.so\.1$' | tr -d ' ' | cut -d '>' -f 2)"

examples/simpleapp/README.md

@@ -19,12 +19,12 @@ Otherwise, to run SimpleApp outside the IDE:
     * (Windows)
         ```
         cd Sidechains-SDK\examples\simpleapp
-        java -cp ./target/sidechains-sdk-simpleapp-0.3.0.jar;./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+        java -cp ./target/sidechains-sdk-simpleapp-0.3.1.jar;./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
         ```
     * (Linux)
         ```
         cd ./Sidechains-SDK/examples/simpleapp
-        java -cp ./target/sidechains-sdk-simpleapp-0.3.0.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+        java -cp ./target/sidechains-sdk-simpleapp-0.3.1.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
         ```
         On some Linux OSs during backward transfers certificates proofs generation a extremely big RAM consumption may happen, that will lead to the process force killing by the OS.
 
@@ -36,7 +36,7 @@ While we keep monitoring the memory footprint of the proofs generation process,
 	     - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows:
 
 	     ```
-	     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.0.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+	     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.1.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
 	     ```
 	      - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node.
 

examples/simpleapp/mc_sc_workflow_example.md

@@ -15,7 +15,7 @@ Build SDK components by using command (in the root of the SDK folder):
 
 Run Bootstrapping tool using command: 
 
-`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.0.jar`
+`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.1.jar`
 
 All other commands are performed as commands for Bootstrapping tool in next format: `"command name"  "parameters for command in JSON format"`.
 For any help you could use command `help`, for exit just print `exit`
@@ -397,15 +397,15 @@ Run SimpleApp with the `my_settings.conf`:
 
 * For Windows:
  ```
-    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.0.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
+    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.1.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
 ```
 * For Linux (Glibc):
 ```
-    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.0.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
+    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.1.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
 ```
 * For Linux (Jemalloc):
 ```
-    LD_PRELOAD=<path to jemalloc library>/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.0.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
+    LD_PRELOAD=<path to jemalloc library>/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.1.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
 ```
 - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node.
 

examples/simpleapp/pom.xml

@@ -2,7 +2,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>io.horizen</groupId>
 	<artifactId>sidechains-sdk-simpleapp</artifactId>
-	<version>0.3.0</version>
+	<version>0.3.1</version>
 	<inceptionYear>2018</inceptionYear>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -16,7 +16,7 @@
 		<dependency>
 			<groupId>io.horizen</groupId>
 			<artifactId>sidechains-sdk</artifactId>
-			<version>0.3.0</version>
+			<version>0.3.1</version>
 		</dependency>
 
 		<dependency>

pom.xml

@@ -2,7 +2,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.horizen</groupId>
 	<artifactId>Sidechains</artifactId>
-	<version>0.3.0</version>
+	<version>0.3.1</version>
 	<inceptionYear>2018</inceptionYear>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

qa/SidechainTestFramework/sc_test_framework.py

@@ -114,7 +114,7 @@ def main(self):
                           help="Don't stop bitcoinds after the test execution")
         parser.add_option("--zendir", dest="zendir", default="ZenCore/src",
                           help="Source directory containing zend/zen-cli (default: %default)")
-        parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.0.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future
+        parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.1.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future
                           help="Directory containing .jar file for SC (default: %default)")
         parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="sc_test"),
                           help="Root directory for datadirs")

qa/SidechainTestFramework/scutil.py

@@ -121,7 +121,7 @@ def launch_bootstrap_tool(command_name, json_parameters):
     json_param = json.dumps(json_parameters)
     java_ps = subprocess.Popen(["java", "-jar",
                                 os.getenv("SIDECHAIN_SDK",
-                                          "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.0.jar",
+                                          "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.1.jar",
                                 command_name, json_param], stdout=subprocess.PIPE)
     sc_bootstrap_output = java_ps.communicate()[0]
     try:
@@ -448,7 +448,7 @@ def start_sc_node(i, dirname, extra_args=None, rpchost=None, timewait=None, bina
         lib_separator = ";"
 
     if binary is None:
-        binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.0.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp"
+        binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.1.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp"
     #        else if platform.system() == 'Linux':
     '''
     In order to effectively attach a debugger (e.g IntelliJ) to the simpleapp, it is necessary to start the process

qa/run_sc_tests.py

@@ -13,6 +13,7 @@
 from sc_multiple_certs import SCMultipleCerts
 from sc_nodes_initialize import SidechainNodesInitializationTest
 from sc_versions_and_mc_certs import SCVersionsAndMCCertificates
+from sc_withdrawal_epoch_last_block import SCWithdrawalEpochLastBlock
 from test_framework.util import assert_equal
 from mc_sc_connected_nodes import MCSCConnectedNodes
 from mc_sc_forging1 import MCSCForging1
@@ -75,6 +76,9 @@ def run_tests(log_file):
     result = run_test(SCForwardTransfer())
     assert_equal(0, result, "sc_forward_transfer test failed!")
 
+    result = run_test(SCWithdrawalEpochLastBlock())
+    assert_equal(0, result, "sc_withdrawal_epoch_last_block test failed!")
+
     result = run_test(SCCumCommTreeHash())
     assert_equal(0, result, "sc_cum_comm_tree_hash test failed!")
 

qa/sc_withdrawal_epoch_last_block.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+
+from SidechainTestFramework.sc_boostrap_info import SCNodeConfiguration, SCCreationInfo, MCConnectionInfo, \
+    SCNetworkConfiguration
+from SidechainTestFramework.sc_test_framework import SidechainTestFramework
+from test_framework.util import assert_equal, assert_true, start_nodes, \
+    websocket_port_by_mc_node_index, forward_transfer_to_sidechain
+from SidechainTestFramework.scutil import bootstrap_sidechain_nodes, \
+    start_sc_nodes, is_mainchain_block_included_in_sc_block, \
+    check_mainchain_block_reference_info, generate_next_block, generate_next_blocks
+
+"""
+Check that in the SC block, that contains MC block ref leading to the withdrawal epoch end, we allow to have MC2SCAggTx,
+but not SC2SC transactions.
+
+Configuration:
+    Start 1 MC node and 1 SC node (with default websocket configuration).
+    SC node connected to the first MC node.
+
+Test:
+    For the SC node:
+        - Mine MC blocks till one block before the withdrawal epoch end.
+        - Forge SC blocks to sync with MC.
+        - Create new forward transfer to sidechain.
+        - Mine MC block that will have a FT.
+        - Generate CoreTx on SC node.
+        - Try to generate SC block that will end with MC block ref with AggTx, but without Sc2Sc tx.
+        - Check that Sc2Sc tx is still in the mempool.
+"""
+class SCWithdrawalEpochLastBlock(SidechainTestFramework):
+    sc_nodes_bootstrap_info=None
+    withdrawal_epoch_length=5
+
+    def setup_nodes(self):
+        return start_nodes(1, self.options.tmpdir)
+
+    def sc_setup_chain(self):
+        mc_node = self.nodes[0]
+        sc_node_configuration = SCNodeConfiguration(
+            MCConnectionInfo(address="ws://{0}:{1}".format(mc_node.hostname, websocket_port_by_mc_node_index(0)))
+        )
+        network = SCNetworkConfiguration(SCCreationInfo(mc_node, 100, self.withdrawal_epoch_length), sc_node_configuration)
+        self.sc_nodes_bootstrap_info = bootstrap_sidechain_nodes(self.options, network)
+
+    def sc_setup_nodes(self):
+        return start_sc_nodes(1, self.options.tmpdir)
+
+    def run_test(self):
+        mc_node = self.nodes[0]
+        sc_node = self.sc_nodes[0]
+        withdrawal_epoch_blocks_left = self.withdrawal_epoch_length - 1
+
+        # Send some coins to SC node wallet.
+        mc_return_address = mc_node.getnewaddress()
+        (sc_info, mc_block_count) = forward_transfer_to_sidechain(self.sc_nodes_bootstrap_info.sidechain_id,
+                                                                  mc_node,
+                                                                  self.sc_nodes_bootstrap_info.genesis_account.publicKey,
+                                                                  self.sc_nodes_bootstrap_info.genesis_account_balance,
+                                                                  mc_return_address,
+                                                                  generate_block=True)
+
+        generate_next_blocks(sc_node, "first node", 1)
+        withdrawal_epoch_blocks_left -= 1
+
+        # Check the MC block reference's inclusion
+        sc_best_block = sc_node.block_best()["result"]
+        mc_block = self.nodes[0].getblock(str(mc_block_count))
+
+        res = is_mainchain_block_included_in_sc_block(sc_best_block["block"], mc_block)
+        assert_true(res, "The mainchain block is not included in SC node.")
+
+        sc_mc_best_block_ref_info = sc_node.mainchain_bestBlockReferenceInfo()["result"]
+        assert_true(
+            check_mainchain_block_reference_info(sc_mc_best_block_ref_info, mc_block),
+            "The mainchain block is not included inside SC block reference info.")
+
+        # Generate more MC blocks to reach 1 block before the end of the withdrawal epoch
+        mc_node.generate(withdrawal_epoch_blocks_left - 1)
+
+        # Generate SC block
+        generate_next_blocks(sc_node, "first node", 1)
+
+        # Send 1 more FT in the withdrawal epoch last MC block.
+        (sc_info, mc_block_count) = forward_transfer_to_sidechain(self.sc_nodes_bootstrap_info.sidechain_id,
+                                                                  mc_node,
+                                                                  self.sc_nodes_bootstrap_info.genesis_account.publicKey,
+                                                                  self.sc_nodes_bootstrap_info.genesis_account_balance,
+                                                                  mc_return_address,
+                                                                  generate_block=True)
+
+        # Create SC to SC tx on SC node
+        fee = 10
+        sendCoins = {
+            "outputs": [
+                {
+                    "publicKey": self.sc_nodes_bootstrap_info.genesis_account.publicKey,
+                    "value": self.sc_nodes_bootstrap_info.genesis_account_balance - fee
+                }
+            ],
+            "fee": fee,
+        }
+        sc_node.transaction_sendCoinsToAddress(json.dumps(sendCoins))
+
+        # Check mempool
+        assert_equal(1, len(sc_node.transaction_allTransactions()["result"]["transactions"]),
+                     "FT spending Tx expected to be in the SC node mempool.")
+
+        # Generate 1 more SC block and check that we have 1 MC block ref with AggTx and no SC2SCTx
+        generate_next_block(sc_node, "first node")
+        sc_best_block = sc_node.block_best()["result"]
+
+        # Check that there are no SC2SC txs
+        assert_equal(0, len(sc_best_block["block"]["sidechainTransactions"]), "No sidechain transactions expected.")
+
+        # Check the MC block reference's inclusion
+        mc_block = self.nodes[0].getblock(str(mc_block_count))
+
+        res = is_mainchain_block_included_in_sc_block(sc_best_block["block"], mc_block)
+        assert_true(res, "The mainchain block is not included in SC node.")
+
+        sc_mc_best_block_ref_info = sc_node.mainchain_bestBlockReferenceInfo()["result"]
+        assert_true(
+            check_mainchain_block_reference_info(sc_mc_best_block_ref_info, mc_block),
+            "The mainchain block is not included inside SC block reference info.")
+
+        # Check mempool if SC 2 SC tx is still present
+        assert_equal(1, len(sc_node.transaction_allTransactions()["result"]["transactions"]),
+                     "FT spending Tx expected to be in the SC node mempool.")
+
+
+if __name__ == "__main__":
+    SCWithdrawalEpochLastBlock().main()

sdk/pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>io.horizen</groupId>
   <artifactId>sidechains-sdk</artifactId>
-  <version>0.3.0</version>
+  <version>0.3.1</version>
   <name>${project.groupId}:${project.artifactId}</name>
   <description>Zendoo is a unique sidechain and scaling solution developed by Horizen. The Zendoo ${project.artifactId} is a framework that supports the creation of sidechains and their custom business logic, with the Horizen public blockchain as the mainchain.</description>
   <url>https://github.com/${project.github.organization}/${project.artifactId}</url>

sdk/src/main/java/com/horizen/proof/AbstractSignature25519.java

@@ -15,7 +15,7 @@ public abstract class AbstractSignature25519<S extends PrivateKey25519, P extend
         implements ProofOfKnowledge<S, P> {
 
     @JsonProperty("signature")
-    final byte[] signatureBytes;
+    protected final byte[] signatureBytes;
 
     public AbstractSignature25519(byte[] signatureBytes) {
         this.signatureBytes = Arrays.copyOf(signatureBytes, signatureBytes.length);

sdk/src/main/scala/com/horizen/block/SidechainBlock.scala

@@ -51,18 +51,9 @@ class SidechainBlock(override val header: SidechainBlockHeader,
 
   override lazy val id: ModifierId = header.id
 
-  // TODO: prettify
   override lazy val transactions: Seq[SidechainTypes#SCBT] = {
-    var txs = Seq[SidechainTypes#SCBT]()
-
-    for(b <- mainchainBlockReferencesData) {
-      if (b.sidechainRelatedAggregatedTransaction.isDefined) {
-        txs = txs :+ b.sidechainRelatedAggregatedTransaction.get
-      }
-    }
-    for(tx <- sidechainTransactions)
-      txs = txs :+ tx.asInstanceOf[SidechainTypes#SCBT]
-    txs
+    mainchainBlockReferencesData.flatMap(_.sidechainRelatedAggregatedTransaction) ++
+      sidechainTransactions
   }
 
   def feePaymentsHash: Array[Byte] = header.feePaymentsHash

sdk/src/main/scala/com/horizen/validation/WithdrawalEpochValidator.scala

@@ -71,14 +71,14 @@ class WithdrawalEpochValidator(params: NetworkParams) extends HistoryBlockValida
             throw new IllegalArgumentException("Sidechain block %s contains MC Block references, that belong to different withdrawal epochs.".format(BytesUtils.toHexString(idToBytes(block.id))))
 
         } else { // epoch is the same
-          if (blockEpochInfo.lastEpochIndex == params.withdrawalEpochLength && block.transactions.nonEmpty) // Block is the last block of the epoch and contains SC Txs
-            throw new IllegalArgumentException("Sidechain block %s is the last withdrawal epoch block, but contains Sidechain Transactions.".format(BytesUtils.toHexString(idToBytes(block.id))))
+          // Block is the last block of the withdrawal epoch and contains SC2SC Txs.
+          // Note: MC2SCAggTx is allowed, because of being a part of MC block reference data.
+          if (blockEpochInfo.lastEpochIndex == params.withdrawalEpochLength && block.sidechainTransactions.nonEmpty)
+            throw new IllegalArgumentException("Sidechain block %s is the withdrawal epoch last block, but contains Sidechain Transactions.".format(BytesUtils.toHexString(idToBytes(block.id))))
         }
 
       case None =>
         throw new IllegalArgumentException("Sidechain block %s parent block is missed.".format(BytesUtils.toHexString(idToBytes(block.id))))
     }
-
-    val backwardTransferCertificateCount = block.mainchainBlockReferencesData.flatMap(_.topQualityCertificate).size
   }
 }

sdk/src/test/scala/com/horizen/fixtures/MainchainBlockReferenceFixture.scala

@@ -2,11 +2,12 @@ package com.horizen.fixtures
 
 import java.time.Instant
 import java.util.Random
-
 import com.horizen.block.{MainchainBlockReference, MainchainBlockReferenceData, MainchainHeader, SidechainBlock}
 import com.horizen.chain.{MainchainHeaderHash, byteArrayToMainchainHeaderHash}
 import com.horizen.params.NetworkParams
+import com.horizen.transaction.MC2SCAggregatedTransaction
 import com.horizen.utils._
+import org.scalatestplus.mockito.MockitoSugar.mock
 
 import scala.annotation.tailrec
 import scala.collection.mutable
@@ -113,4 +114,11 @@ trait MainchainBlockReferenceFixture extends MainchainHeaderFixture {
       generated
     }
   }
+
+
+  def mainchainBlockReferenceWithMockedAggTx(ref: MainchainBlockReference): MainchainBlockReference = {
+    new MainchainBlockReference(ref.header, MainchainBlockReferenceData(ref.header.hash, Some(mock[MC2SCAggregatedTransaction]), None, None, Seq(), None)) {
+      override def semanticValidity(params: NetworkParams): Try[Unit] = Success(Unit)
+    }
+  }
 }