Merge pull request #404 from HorizenOfficial/snark_keys_generation_fix

Release 0.3.5: Snark keys generation fix

CHANGELOG.md

@@ -1,3 +1,6 @@
+**0.3.5**
+1. Snark keys generation fixed: circuit specific segment size added.
+
 **0.3.4**
 1. Added the possibility to perform a backup of a sidechain non coin-boxes and restore these boxes into a new bootstrapped sidechain of the same type.
 2. log4j version updated.

README.md

@@ -40,7 +40,7 @@ While we keep monitoring the memory footprint of the proofs generation process,
  - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows:
 
 ```
-LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.5.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
 ```
  - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. 
 

ci/run_sc.sh

@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.4}"
+SIMPLE_APP_VERSION="${SIMPLE_APP_VERSION:-0.3.5}"
 
 if [ -d "$1" ] && [ -f "$2" ]; then
   path_to_jemalloc="$(ldconfig -p | grep "$(arch)" | grep 'libjemalloc\.so\.1$' | tr -d ' ' | cut -d '>' -f 2)"

examples/simpleapp/README.md

@@ -19,12 +19,12 @@ Otherwise, to run SimpleApp outside the IDE:
     * (Windows)
         ```
         cd Sidechains-SDK\examples\simpleapp
-        java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar;./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+        java -cp ./target/sidechains-sdk-simpleapp-0.3.5.jar;./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
         ```
     * (Linux)
         ```
         cd ./Sidechains-SDK/examples/simpleapp
-        java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+        java -cp ./target/sidechains-sdk-simpleapp-0.3.5.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
         ```
         On some Linux OSs during backward transfers certificates proofs generation a extremely big RAM consumption may happen, that will lead to the process force killing by the OS.
 
@@ -36,7 +36,7 @@ While we keep monitoring the memory footprint of the proofs generation process,
 	     - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows:
 
 	     ```
-	     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.4.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+	     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.5.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
 	     ```
 	      - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node.
 

examples/simpleapp/mc_sc_workflow_example.md

@@ -15,7 +15,7 @@ Build SDK components by using command (in the root of the SDK folder):
 
 Run Bootstrapping tool using command: 
 
-`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.4.jar`
+`java -jar tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.5.jar`
 
 All other commands are performed as commands for Bootstrapping tool in next format: `"command name"  "parameters for command in JSON format"`.
 For any help you could use command `help`, for exit just print `exit`
@@ -397,15 +397,15 @@ Run SimpleApp with the `my_settings.conf`:
 
 * For Windows:
  ```
-    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
+    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.5.jar;./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
 ```
 * For Linux (Glibc):
 ```
-    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
+    java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.5.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
 ```
 * For Linux (Jemalloc):
 ```
-    LD_PRELOAD=<path to jemalloc library>/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
+    LD_PRELOAD=<path to jemalloc library>/libjemalloc.so.1 java -cp ./examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.5.jar:./examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp ./examples/my_settings.conf
 ```
 - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node.
 

examples/simpleapp/pom.xml

@@ -2,7 +2,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>io.horizen</groupId>
 	<artifactId>sidechains-sdk-simpleapp</artifactId>
-	<version>0.3.4</version>
+	<version>0.3.5</version>
 	<inceptionYear>2018</inceptionYear>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -16,7 +16,7 @@
 		<dependency>
 			<groupId>io.horizen</groupId>
 			<artifactId>sidechains-sdk</artifactId>
-			<version>0.3.4</version>
+			<version>0.3.5</version>
 		</dependency>
 
 		<dependency>

pom.xml

@@ -2,7 +2,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.horizen</groupId>
 	<artifactId>Sidechains</artifactId>
-	<version>0.3.4</version>
+	<version>0.3.5</version>
 	<inceptionYear>2018</inceptionYear>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

qa/SidechainTestFramework/sc_boostrap_info.py

@@ -23,12 +23,15 @@ class SCCreationInfo(object):
     # Note: the maximum withdrawal_epoch_length allowed is around 900, otherwise snark keys size check will fail
     # because of too complex circuit from MC perspective.
     def __init__(self, mc_node, forward_amount=100, withdrawal_epoch_length=LARGE_WITHDRAWAL_EPOCH_LENGTH,
-                 btr_data_length=0, sc_creation_version=SC_CREATION_VERSION_1):
+                 btr_data_length=0, sc_creation_version=SC_CREATION_VERSION_1,
+                 cert_max_keys=7, cert_sig_threshold=5):
         self.mc_node = mc_node
         self.forward_amount = forward_amount
         self.withdrawal_epoch_length = withdrawal_epoch_length
         self.btr_data_length = btr_data_length
         self.sc_creation_version = sc_creation_version
+        self.cert_max_keys = cert_max_keys
+        self.cert_sig_threshold = cert_sig_threshold
 
 
 """

qa/SidechainTestFramework/sc_test_framework.py

@@ -114,7 +114,7 @@ def main(self):
                           help="Don't stop bitcoinds after the test execution")
         parser.add_option("--zendir", dest="zendir", default="ZenCore/src",
                           help="Source directory containing zend/zen-cli (default: %default)")
-        parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future
+        parser.add_option("--scjarpath", dest="scjarpath", default="../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.5.jar;../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp", #New option. Main class path won't be needed in future
                           help="Directory containing .jar file for SC (default: %default)")
         parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="sc_test"),
                           help="Root directory for datadirs")

qa/SidechainTestFramework/scutil.py

@@ -120,7 +120,7 @@ def launch_bootstrap_tool(command_name, json_parameters):
     json_param = json.dumps(json_parameters)
     java_ps = subprocess.Popen(["java", "-jar",
                                 os.getenv("SIDECHAIN_SDK",
-                                          "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.4.jar",
+                                          "..") + "/tools/sctool/target/sidechains-sdk-scbootstrappingtools-0.3.5.jar",
                                 command_name, json_param], stdout=subprocess.PIPE)
     sc_bootstrap_output = java_ps.communicate()[0]
     try:
@@ -142,7 +142,7 @@ def launch_db_tool(dirName, command_name, json_parameters):
     json_param = json.dumps(json_parameters)
     java_ps = subprocess.Popen(["java", "-jar",
                                 os.getenv("SIDECHAIN_SDK",
-                                          "..") + "/tools/dbtool/target/sidechains-sdk-dbtools-0.3.4.jar",
+                                          "..") + "/tools/dbtool/target/sidechains-sdk-dbtools-0.3.5.jar",
                                 storagesPath, command_name, json_param], stdout=subprocess.PIPE)
     db_tool_output = java_ps.communicate()[0]
     try:
@@ -468,7 +468,7 @@ def start_sc_node(i, dirname, extra_args=None, rpchost=None, timewait=None, bina
         lib_separator = ";"
 
     if binary is None:
-        binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.4.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp"
+        binary = "../examples/simpleapp/target/sidechains-sdk-simpleapp-0.3.5.jar" + lib_separator + "../examples/simpleapp/target/lib/* com.horizen.examples.SimpleApp"
     #        else if platform.system() == 'Linux':
     '''
     In order to effectively attach a debugger (e.g IntelliJ) to the simpleapp, it is necessary to start the process
@@ -777,7 +777,7 @@ def bootstrap_sidechain_nodes(options, network=SCNetworkConfiguration, block_tim
     ps_keys_dir = os.getenv("SIDECHAIN_SDK", "..") + "/qa/ps_keys"
     if not os.path.isdir(ps_keys_dir):
         os.makedirs(ps_keys_dir)
-    cert_keys_paths = cert_proof_keys_paths(ps_keys_dir)
+    cert_keys_paths = cert_proof_keys_paths(ps_keys_dir, sc_creation_info.cert_max_keys)
     csw_keys_paths = csw_proof_keys_paths(ps_keys_dir, sc_creation_info.withdrawal_epoch_length)
     sc_nodes_bootstrap_info = create_sidechain(sc_creation_info,
                                                block_timestamp_rewind,
@@ -807,10 +807,11 @@ def bootstrap_sidechain_nodes(options, network=SCNetworkConfiguration, block_tim
     return sc_nodes_bootstrap_info
 
 
-def cert_proof_keys_paths(dirname):
+def cert_proof_keys_paths(dirname, cert_threshold_sig_max_keys):
     # use replace for Windows OS to be able to parse the path to the keys in the config file
-    return ProofKeysPaths(os.path.join(dirname, "cert_marlin_snark_pk").replace("\\", "/"),
-                          os.path.join(dirname, "cert_marlin_snark_vk").replace("\\", "/"))
+    return ProofKeysPaths(
+        os.path.join(dirname, "cert_marlin_snark_pk_" + str(cert_threshold_sig_max_keys)).replace("\\", "/"),
+        os.path.join(dirname, "cert_marlin_snark_vk_" + str(cert_threshold_sig_max_keys)).replace("\\", "/"))
 
 
 def csw_proof_keys_paths(dirname, withdrawal_epoch_length):
@@ -836,7 +837,7 @@ def create_sidechain(sc_creation_info, block_timestamp_rewind, cert_keys_paths,
     vrf_keys = generate_vrf_secrets("seed", 1)
     genesis_account = accounts[0]
     vrf_key = vrf_keys[0]
-    certificate_proof_info = generate_certificate_proof_info("seed", 7, 5, cert_keys_paths)
+    certificate_proof_info = generate_certificate_proof_info("seed", sc_creation_info.cert_max_keys, sc_creation_info.cert_sig_threshold, cert_keys_paths)
     csw_verification_key = generate_csw_proof_info(sc_creation_info.withdrawal_epoch_length, csw_keys_paths)
     genesis_info = initialize_new_sidechain_in_mainchain(
         sc_creation_info.mc_node,

qa/run_sc_tests.py

@@ -8,6 +8,7 @@
 from sc_cert_submitter_after_sync_1 import ScCertSubmitterAfterSync1
 from sc_cert_submitter_after_sync_2 import ScCertSubmitterAfterSync2
 from sc_csw_ceased_at_epoch_1 import SCCswCeasedAtEpoch1
+from sc_csw_ceased_at_epoch_1_with_large_epoch_length import SCCswCeasedAtEpoch1WithLargeEpochLength
 from sc_csw_ceased_at_epoch_2 import SCCswCeasedAtEpoch2
 from sc_csw_ceased_at_epoch_3 import SCCswCeasedAtEpoch3
 from sc_cum_comm_tree_hash import SCCumCommTreeHash
@@ -122,6 +123,9 @@ def run_tests(log_file):
     result = run_test(SCCswCeasedAtEpoch1())
     assert_equal(0, result, "sc_csw_ceased_at_epoch_1 test failed!")
 
+    result = run_test(SCCswCeasedAtEpoch1WithLargeEpochLength())
+    assert_equal(0, result, "sc_csw_ceased_at_epoch_1_with_large_epoch_length test failed!")
+
     result = run_test(SCCswCeasedAtEpoch2())
     assert_equal(0, result, "sc_csw_ceased_at_epoch_2 test failed!")
 

qa/sc_backward_transfer.py

@@ -46,11 +46,19 @@ def setup_nodes(self):
         return start_nodes(num_nodes, self.options.tmpdir, extra_args=[['-debug=sc', '-debug=ws',  '-logtimemicros=1', '-scproofqueuesize=0']] * num_nodes)
 
     def sc_setup_chain(self):
+        # After bug spotted in 0.3.4 we test certificate generation with max keys number > 8
+        cert_max_keys = 10
+        cert_sig_threshold = 6
+
         mc_node = self.nodes[0]
         sc_node_configuration = SCNodeConfiguration(
-            MCConnectionInfo(address="ws://{0}:{1}".format(mc_node.hostname, websocket_port_by_mc_node_index(0)))
+            MCConnectionInfo(address="ws://{0}:{1}".format(mc_node.hostname, websocket_port_by_mc_node_index(0))),
+            submitter_private_keys_indexes=list(range(cert_max_keys))  # SC node owns all schnorr private keys.
         )
-        network = SCNetworkConfiguration(SCCreationInfo(mc_node, 100, self.sc_withdrawal_epoch_length), sc_node_configuration)
+
+        network = SCNetworkConfiguration(SCCreationInfo(mc_node, 100, self.sc_withdrawal_epoch_length,
+                                                        cert_max_keys=cert_max_keys,
+                                                        cert_sig_threshold=cert_sig_threshold), sc_node_configuration)
         self.sc_nodes_bootstrap_info = bootstrap_sidechain_nodes(self.options, network)
 
     def sc_setup_nodes(self):