Merge pull request #436 from HorizenOfficial/dev

0.4.0 Release

.gitignore

@@ -1,3 +1,4 @@
+pom.xml.versionsBackup
 out/
 .idea/
 target/
@@ -13,3 +14,7 @@ examples/simpleapp/dependency-reduced-pom.xml
 qa/tmp
 hs_err*
 log
+/examples/entrypoint.sh
+/examples/Dockerfile
+project/
+

.travis.yml

@@ -1,8 +1,8 @@
 os: linux
-dist: bionic
-language: generic
+dist: focal
 vm:
   size: large
+language: generic
 addons:
   apt:
     packages:
@@ -12,14 +12,15 @@ services:
 
 env:
   global:
-    - CONTAINER_JAVA_VER="openjdk-11-jdk-headless"
-    - CONTAINER_SCALA_VER="2.12.12"
-    - CONTAINER_SCALA_DEB_SHA256SUM="7ecbc3850d8186c0084be37c01cdd987e97328fdd74eb781bf6dc050dba95276"
+    - DOCKER_ORG=zencash
+    - IMAGE_NAME=sc-ci-base
+
+script:
+  - source ci/setup_env.sh
+  - ci/travis_keep_alive.sh
+  - ci/script.sh
 
-script: >-
-  source ci/setup_env.sh &&
-  docker build --pull --no-cache -t zencash/sidechains-sdk-builder ./ci/docker_ci_builder/ &&
-  bash -c "docker run --rm -v $(pwd):/build -v ${HOME}/key.asc:/key.asc --tmpfs /tmp:uid=$(id -u),gid=$(id -g),exec,mode=1777 \
-    --tmpfs /run:uid=$(id -u),gid=$(id -g),exec,mode=1777 -e LOCAL_USER_ID=$(id -u) -e LOCAL_GRP_ID=$(id -g) \
-    $(env | grep -E '^CONTAINER_' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' ') \
-    zencash/sidechains-sdk-builder /build/ci/start_ci.sh"
+jobs:
+  include:
+    - name: jdk-11_latest
+      env: IMAGE_TAG='bionic_jdk-11_latest' TESTS='000,001'

CHANGELOG.md

@@ -1,3 +1,19 @@
+**0.4.0**
+1. Memory pool improvement: upper bound size limit introduced (default 300Mb) altogether with cleanup strategy (the lowest fee rate transaction removed first).
+2. Memory pool improvement: minimum fee rate check added for incoming transactions. By default, is disabled.
+3. CSW is optional. Sidechains without CSW support are available now. Note: keep using CSW feature in real environment to have a possibility to withdraw coins in case of ceasing.
+4. API Authorization added to the coin critical endpoints, like keys management, transaction creation and submission, csw creation, etc.
+5. API freezing during node synchronization resolved.
+6. New API endpoints added for importing/exporting keys to/from the wallet.
+7. Forger sorts transactions by fee rate instead of fee.
+8. Wallet: max fee check added for locally generated transactions to prevent absurdly high fees. Max fee value is set in zennies. Default value is 10000000 (1 Zen).
+9. Custom propositions wallet management improved: complex multi-key propositions are now recognized.
+10. Peers spam detection mechanism improved: "trash" data detection in the end of the block/transaction added.
+11. Bootstrapping tool: dlog keys multiple initialization prevented.
+12. Extra verbosity added to the API responses.
+13. Logging system improved. Application specific configs are allowed.
+14. FeePayments visibility bug fixed: wrongly added fee payments to the block info when there were no payment at all.
+
 **0.3.5**
 1. Snark keys generation fixed: circuit specific segment size added.
 
@@ -37,7 +53,7 @@
 **Beta changes**
 1. Mainchain synchronization: added backward transfer support with Withdrawal certificate with threshold signature zero-knowledge proof by using [zendoo-sc-cryptolib](https://github.com/HorizenOfficial/zendoo-sc-cryptolib)
 2. Added [Latus Proof-of-Stake consensus protocol](https://www.horizen.global/assets/files/Horizen-Sidechain-Zendoo-A_zk-SNARK-Verifiable-Cross-Chain-Transfer-Protocol.pdf)  for sidechain based on [Ouroboros Praos](https://eprint.iacr.org/2017/573.pdf) consensus protocol which supporting forks in Sidechain and Mainchain. Autoforging for Sidechain node is added as well.
-3. Reworked Transactions structure: introduced SidechainCoreTranscation, Boxes structure was improved/changed now three types of boxes are present by default: zen box, withdrawal request box and forger box. Forger box is used for consensus forger selection.
+3. Reworked Transactions structure: introduced SidechainCoreTransaction, Boxes structure was improved/changed now three types of boxes are present by default: zen box, withdrawal request box and forger box. Forger box is used for consensus forger selection.
 
 
 **Alpha features**

README.md

@@ -40,7 +40,7 @@ While we keep monitoring the memory footprint of the proofs generation process,
  - After the installation, just run `export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` before starting the sidechain node, or run the sidechain node adding `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1` at the beginning of the java command line as follows:
 
 ```
-LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.3.5.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
+LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 java -cp ./target/sidechains-sdk-simpleapp-0.4.0.jar:./target/lib/* com.horizen.examples.SimpleApp <path_to_config_file>
 ```
  - In the folder `ci` you will find the script `run_sc.sh` to automatically check and use jemalloc library while starting the sidechain node. 
 
@@ -63,3 +63,7 @@ The project has a Maven module structure and consists of 4 modules:
 **Examples**
 
 You can find an example of a sidechain implementation without any custom business logic here: [Simple App](examples/simpleapp/README.md). A detailed description of how to set up and run a sidechain node with a connection to the mainchain [can be found here](examples/simpleapp/mc_sc_workflow_example.md).
+
+## Extras
+
+In order to build and use `SNAPSHOT.jar` package version refer to the following [documentation](./ci/README.md)

ci/README.md

@@ -0,0 +1,58 @@
+# New version build functionality
+
+As of 2022/06/27 CI/CD pipeline adds functionality to build and publish multiple `DIGIT.DIGIT.DIGIT-SNAPSHOT` versions of `zendoo-sc-crypotolib` package
+with the help of set_version.sh script.
+
+`set_version.sh` script is located under **ci/devtools** directory and automates preparation steps for building/releasing a new
+version of the artifacts by setting the provided version for all the required dependencies across the configuration files.
+
+---
+## Prerequisites for publishing a package:
+  - Singed by GPG key commit and valid GitHub tag in the format of `DIGIT.DIGIT.DIGIT` or `DIGIT.DIGIT.DIGIT-SNAPSHOT`
+  - GitHub tag matching `${pom_version_of_package}"[0-9]*$` regex
+  - Your(a person who pushes a tag) GPG key being added to CI/CD pipeline build settings
+
+Otherwise, the build process will run without entering the publishing stage.
+
+`DIGIT.DIGIT.DIGIT-SNAPSHOT` package version can be built multiple times by adjusting GitHub tag name accordingly. For example:
+```
+GitHub tag = 1.1.1-SNAPSHOT can build 1.1.1-SNAPSHOT package
+GitHub tag = 1.1.1-SNAPSHOT1 can build 1.1.1-SNAPSHOT package
+GitHub tag = 1.1.1-SNAPSHOT2 can build 1.1.1-SNAPSHOT package
+```
+All SNAPSHOT packages are being pushed to a snapshot repository configured under pom.xml file:
+```
+<snapshotRepository>
+      <id>ossrh</id>
+      <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+</snapshotRepository>
+```
+and can be referred to inside the configuration files by providing the full version, that can be found inside nexus [repository](https://oss.sonatype.org/content/repositories/snapshots/io/horizen/sidechains-sdk/)
+
+---
+## Usage
+Before starting the build process use `set_version.sh` script if needed by providing two arguments in the following format:
+```
+ ./ci/devtools/set_version.sh --help
+  Usage: Provide OLD and NEW versions as the 1st and 2nd arguments respectively.
+         It has to match the following format:
+         DIGIT.DIGIT.DIGIT or DIGIT.DIGIT.DIGIT-SNAPSHOT
+
+         For example:
+         ./set_version.sh 5.5.5 5.5.5-SNAPSHOT
+         ./set_version.sh 5.5.5-SNAPSHOT 5.5.5
+```
+| Changes made by set_version.sh script need to be committed before the build. |
+|------------------------------------------------------------------------------|
+
+---
+## How to refer
+- Find all the existing versions of [0.3.5-SNAPSHOT package](https://oss.sonatype.org/content/repositories/snapshots/io/horizen/sidechains-sdk/0.3.5-SNAPSHOT/)
+- Use the full version of SNAPSHOT package as a dependency in the following format for your project.
+```
+<dependency>
+    <groupId>io.horizen</groupId>
+    <artifactId>sidechains-sdk</artifactId>
+    <version>0.3.5-20220609.200517-2</version>
+</dependency>
+```

ci/devtools/.gitignore

@@ -0,0 +1 @@
+*.log

ci/devtools/lint_ci.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." &> /dev/null && pwd )"
+
+# travis-cli lint
+cd "$workdir"/ci/devtools/travis-cli
+docker build --pull -t travis-cli .
+docker run --rm -t -v "$workdir:/mnt" travis-cli lint /mnt/.travis.yml
+
+# run_shellcheck
+docker pull koalaman/shellcheck
+cd "$workdir"
+find . -type f -name '*.sh' -print0 | xargs -0 docker run --rm -t -v "$workdir:/mnt" koalaman/shellcheck

ci/devtools/lint_pom.xml.sh

@@ -1,7 +1,11 @@
 #!/bin/bash
 
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.."
-for file in "${ROOT_DIR}/examples/simpleapp/pom.xml" "${ROOT_DIR}/pom.xml" "${ROOT_DIR}/sdk/pom.xml" "${ROOT_DIR}/tools/sctool/pom.xml"; do
-  CONTENT="$(xmllint --format --encode UTF-8 "${file}")"
-  echo "${CONTENT}" > "${file}"
+base_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../..";
+
+for location in "${base_dir}" "${base_dir}/sdk" "${base_dir}/tools/dbtool" "${base_dir}/tools/sctool" "${base_dir}/examples/simpleapp"; do
+  CONTENT="$(xmllint --format --encode UTF-8 "${location}"/pom.xml)"
+  echo "${CONTENT}" > "${location}/pom.xml"
 done
+
+SETTINGS_CONTENT="$(xmllint --format --encode UTF-8 ci/mvn_settings.xml)"
+echo "${SETTINGS_CONTENT}" > ci/mvn_settings.xml

ci/devtools/run_locally.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." &> /dev/null && pwd )"
+mapfile -t globals < <(docker run --rm -v "$workdir":/workdir mikefarah/yq e '.env.global.[]' .travis.yml)
+mapfile -t runners < <(docker run --rm -v "$workdir":/workdir mikefarah/yq e '.jobs.include.[].env' .travis.yml)
+
+for i in "${!runners[@]}"; do
+  # shellcheck disable=SC1090
+  ( source <(for var in "${globals[@]}"; do echo "export $var"; done; echo "export ${runners[$i]}"); \
+  "$workdir"/ci/script.sh 2>&1 | tee -a "$workdir/ci/devtools/job-$i.log" )
+done

ci/devtools/set_version.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+# shellcheck disable=SC2154
+
+set -Eeo pipefail
+
+# Get the directory of the currently executing script and its parent dir
+current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+base_dir="$( dirname "${current_dir%/*}" )"
+
+DOCKER_ORG="${DOCKER_ORG:-zencash}"
+IMAGE_NAME="${IMAGE_NAME:-sc-ci-base}"
+IMAGE_TAG="${IMAGE_TAG:-bionic_rust-1.51.0_jdk-11_latest}"
+image="${DOCKER_ORG}/${IMAGE_NAME}:${IMAGE_TAG}"
+
+have_docker="false"
+command -v docker &> /dev/null && have_docker="true"
+
+# Functions
+define(){ IFS=$'\n' read -r -d '' "${1}" || true; }
+
+# Script content
+define execute << SCRIPT
+#!/bin/bash
+
+set -euo pipefail
+
+# Get the directory of the currently executing script and its parent dir
+base_dir=\$( cd \$( dirname ${BASH_SOURCE[0]} ) >/dev/null 2>&1 && pwd )/../..;
+
+# pom xml file locations
+pom_xml_location=\${base_dir}
+sdk_xml_location=\${base_dir}/sdk
+dbtool_xml_location=\${base_dir}/tools/dbtool
+simpleapp_xml_location=\${base_dir}/examples/simpleapp
+sctool_xml_location=\${base_dir}/tools/sctool
+
+# .sh file(s)
+run_sc_file=\${base_dir}/ci/run_sc.sh
+
+# python file(s)
+scutil_file=\${base_dir}/qa/SidechainTestFramework/scutil.py
+sc_test_framework_file=\${base_dir}/qa/SidechainTestFramework/sc_test_framework.py
+
+# .md files
+mc_sc_workflow_file=\${base_dir}/examples/simpleapp/mc_sc_workflow_example.md
+simpleapp_readme_file=\${base_dir}/examples/simpleapp/README.md
+main_readme_file=\${base_dir}/README.md
+
+# Functions
+function fn_die() {
+  echo -e \${1} >&2
+  exit \${2:-1}
+}
+
+function usage() {
+  cat << BLOCK
+  Usage: Provide OLD and NEW versions as the 1st and 2nd arguments respectively.
+         It has to match the following format:
+         DIGIT.DIGIT.DIGIT or DIGIT.DIGIT.DIGIT-SNAPSHOT
+
+         For example:
+         ./set_version.sh 5.5.5 5.5.5-SNAPSHOT
+         ./set_version.sh 5.5.5-SNAPSHOT 5.5.5
+BLOCK
+  fn_die "Exiting ..."
+}
+
+# Checking for exact amount of arguments as the first step
+if [[ $# -eq 2 ]]; then
+    version_old="${1}"
+    version_new="${2}"
+else
+    usage
+fi
+
+# Checking the format of the versions
+if ! [[ \${version_old} =~ ^[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?$ ]]; then
+  usage
+fi
+
+if ! [[ \${version_new} =~ ^[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?$ ]]; then
+  usage
+fi
+
+# Checking if maven is installed
+if ! command -v mvn >/dev/null; then
+  echo "" && echo "=== Maven needs to be installed!!! ===" && echo ""
+  fn_die "Refer to the official Apache Maven Project: https://maven.apache.org/install.html\nExiting ..."
+fi
+
+## Changing version numbers under pom.xml file(s)
+for dir in \${pom_xml_location} \${sdk_xml_location} \${dbtool_xml_location} \${simpleapp_xml_location} \${sctool_xml_location}; do
+# Checking if OLD version matches with the CURRENT version in pom file(s)
+  cd \${dir}
+  current_pom_version=\$(mvn help:evaluate -q -Dexpression=project.version -DforceStdout 2>/dev/null)
+  if [ \${version_old} != \${current_pom_version} ]; then
+    fn_die "Fix it! The OLD version does not match with CURRENT version under \${dir}/pom.xml file\nCurrent version is: \${current_pom_version}. \nExiting ..."
+  fi
+
+  echo "" && echo "=== Modifying pom file under \${dir} location ===" && echo ""
+
+  mvn versions:set -DnewVersion=\${version_new}
+
+  # shellcheck disable=SC1014
+  if grep -cq '<artifactId>sidechains-sdk</artifactIds>' \${dir}/pom.xml; then
+    mvn versions:use-dep-version -Dincludes=io.horizen:sidechains-sdk -DdepVersion=\${version_new} -DforceVersion=true
+  fi
+done
+
+# Changing version under other file(s)
+# shellcheck disable=SC2001
+version_old_dot_escaped=\$(sed -e 's/\./\\\./g' <<< \${version_old})
+
+for file in \${scutil_file} \${sc_test_framework_file} \${mc_sc_workflow_file} \${simpleapp_readme_file} \${main_readme_file} \${run_sc_file}; do
+  # Checking if OLD version matches with the CURRENT version in pom file(s)
+  if [ \${file} != \${run_sc_file} ]; then
+    if ! grep -cq sidechains.*\${version_old} \${file}; then
+      current_jar_version=\$(grep -Eoh 'sidechains-sdk.*[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?' \${file} | cut -d '-' -f4- | head -n1)
+      fn_die "Fix it! The OLD version does not match with CURRENT version under \${file} file\nCurrent version is: \${current_jar_version}"
+    fi
+  elif [ \${file} = \${run_sc_file} ];then
+    if ! grep -cq SIMPLE_APP_VERSION:-\${version_old} \${file}; then
+      current_jar_version=\$(grep -Eioh 'SIMPLE_APP_VERSION:-[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?' \${file} | cut -d '-' -f2-)
+      fn_die "Fix it! The OLD version does not match with CURRENT version under \${file} file\nCurrent version is: \${current_jar_version}"
+    fi
+  fi
+
+  echo "" && echo "=== Modifying \${file} file ===" && echo ""
+  sed -i s/\${version_old_dot_escaped}/\${version_new}/g \${file}
+done
+
+echo "" && echo "=== DONE ===" && echo ""
+echo -e "OLD version: \${version_old}\nNEW version: \${version_new}"
+
+exit 0
+SCRIPT
+
+## If docker is installed running the script inside a container. Otherwise running inside local bash shell
+cmd='bash'
+if [ "$have_docker" = "true" ]; then
+  echo "" && echo "=== Docker is installed. Running the script inside docker container ===" && echo ""
+  cmd="docker run --rm -i -v ${base_dir}:/build -w /build ${image} ${cmd}"
+fi
+
+${cmd} <<< "${execute}"
+