Merge branch 'master' into testnet_q3_mastery

.gitignore

@@ -43,3 +43,14 @@ native/linux/fastvm
 native/linux/solidity
 
 
+# 3rd party
+3rdParty/libJson/build/
+3rdParty/libnsc/build/
+3rdParty/libnzmq/build/
+lib/libJson.jar
+lib/libnsc.jar
+lib/libnzmq.jar
+
+# test folder
+test_db/
+tmp/

.gitmodules

@@ -1,6 +1,8 @@
 [submodule "aion_fastvm"]
-	path = aion_fastvm
-	url = https://github.com/aionnetwork/aion_fastvm
+    path = aion_fastvm
+    url = https://github.com/aionnetwork/aion_fastvm
+    branch = master
 [submodule "aion_api"]
-	path = aion_api
-	url = https://github.com/aionnetwork/aion_api
+    path = aion_api
+    url = https://github.com/aionnetwork/aion_api
+    branch = master

DOCKER.md

@@ -0,0 +1,199 @@
+# Dockerization
+
+Provides a container that can run the AION kernel. Can also attach a remote debugger if necessary.
+
+## Limitations
+
+Currently this image there does NOT support running CLI commands before the kernel startup. 
+One needs to build a custom image and bake in other CLI commands if necessary (override the container startup script).
+
+## Prerequisites
+
+- docker-compose (docker-compose version 1.21.0, build 5920eb0)
+- docker (>= Docker version 17.12.1-ce, build 7390fc6)
+
+The kernel can be deployed as a container using the Dockerfile present in the root of the repo. 
+The docker image can be built using the `pack_docker` ant target. Local development can
+leverage `ant pack_dev_docker` to build a custom image based on current code base.
+
+## Building
+
+### Dev build
+
+```bash
+ant pack_dev_docker
+```
+
+##### Description
+
+This command will:
+* build the project with `-Dcompile.debug=true`
+* build a custom Docker image using the current code adding java debug libs for remote debugging (this requires
+downloading a version of java different from the one automatically packed by the kernel binary)
+
+This build is conditioned by the `DEV_BUILD` argument in the `Dockerfile` so you can add more development related 
+behaviour by using this flag.
+
+##### Remote debug
+
+If you want to use the remote debugging feature:
+* make sure you expose the correct port where you want to attach which is set in the `supporting-services.yml` 
+and defaults to `6006`:
+```bash
+- JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:6006 -Xms4g
+```  
+
+* run the image as described below in [Running](#Running) and expose the debug port:
+```bash
+...
+-p 6006:6006
+...
+```
+
+**Note**: The image build uses the binary built in the `pack` directory by ant. 
+If you want to use a specific kernel binary download the binary from the official repo, rename it to `aion.tar.bz2` and 
+copy it to the `pack` directory
+
+
+### Release build
+
+#### Build docker image
+
+```bash
+ant pack_docker
+```
+
+This command will create 1 docker image with the tag value as `GITVER` variable from `script/prebuild.sh` which is the
+short commit revision.
+Eg:
+```bash
+aion-core:0.2.8f5317462
+```
+
+#### Push docker image
+
+In order to have the image available for deployments/developers you need to tag and push it to you registry of choice manually.
+
+* tag image according to repo:
+```bash
+docker tag aion-core:<short_commit_revision> <your_docker_registry>/aion-core:<short_commit_revision> 
+```
+
+* push image:
+```bash
+docker push <your_docker_registry>/aion-core:<short_commit_revision>
+```
+
+## Running
+
+You can start your kernel container using the `docker run` command and you can override a few parameters by passing
+environment variables to the command.
+
+Eg:
+
+```bash
+docker run -it \
+-p 8545:8545 \
+-p 8547:8547 \
+-e difficulty="0x1" \
+-e mining="true" \
+-e coinbase_password=p@ss \
+-e java_api_listen_address="0.0.0.0" \
+-e rpc_listen_address="0.0.0.0" \
+-e peer_list="p2p://[email protected]:3333,p2p://[email protected]:4444" \
+-e override_peer_list="true" \
+aion-core:0.2.8.f5317462
+```
+
+**Note**: 
+We wont's support setting options for the script as the container will have to be teared down after
+running and recreated again. This is not feasible in a production environment. Instead of passing options to the startup 
+script we should provide a rpc/java API that will allow the required functionality to be present after kernel startup.
+Until then we can still use that functionality by manually starting a bash session in the container and running
+the commands that we need. 
+
+Eg:
+
+* Start the container as described above
+* In a separate terminal get the container id
+```bash
+docker ps
+``` 
+
+* Start a bash session in the container
+```bash
+docker run -it <contaier_id> bash
+```
+
+* Make sure you are in the `/opt/aion` directory and run your desired commands:
+```bash
+./aion.sh -a list
+```
+
+##### Expose API
+
+```bash
+# Support for access outside of the container
+-e java_api_listen_address="0.0.0.0" \
+-e rpc_listen_address="0.0.0.0" \
+
+# Exposes the kernel API (web3 and java)
+-p 8545:8545 \
+-p 8547:8547 \
+```
+
+##### Override peer list
+
+```bash
+-e peer_list="p2p://[email protected]:3333,p2p://[email protected]:4444" \
+-e override_peer_list="true" \
+```
+
+##### Bootstrap internal mining
+
+**Note**: The override of these variables are mostly for local development/testing purposes. 
+
+```bash
+# This will lower the difficulty for slow machines and generate an account at startup and set it as the coinbase
+-e difficulty="0x1" \
+-e mining="true" \
+-e coinbase_password=p@ss \
+```
+
+
+**Note**: if you built a development image you can override parameters in the `supporting-services.yml` file and run it:
+
+```bash
+docker-compose -f supporting-services.yml up
+```
+
+List of environment variables than can override xml properties:
+
+```bash
+- difficulty
+- coinbase_password
+- rpc_listen_address
+- rpc_listen_port
+- cors_enabled
+- apis_enabled
+- java_api_listen_address
+- java_api_listen_port
+- p2p_listen_address
+- p2p_listen_port
+- discover
+- mining
+- miner_address
+- cpu_mine_threads
+- peer_list
+- override_peer_list
+- log_level_db
+- log_level_vm
+- log_level_gen
+- log_level_api
+- log_level_sync
+- log_level_cons
+- log_file
+- log_path
+```
+
+If you need to override more properties update `override-config.py` to support those properties.

Dockerfile

@@ -0,0 +1,44 @@
+ARG UBUNTU_VER=16.04
+FROM ubuntu:${UBUNTU_VER}
+
+LABEL maintainers="[email protected], [email protected]"
+
+# prepare for java instalation
+RUN apt-get update && apt-get install -y bzip2 lsb-release wget curl jq locales net-tools libicu-dev libedit-dev
+RUN apt-get clean
+
+# change locales to UTF-8 in order to avoid bug when changing config.xml
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+WORKDIR /opt
+
+# TODO: use java 10.0.2
+ARG DEV_BUILD=false
+RUN if [ "${DEV_BUILD}" = "true" ]; then wget -O jdk.tar.gz \
+https://download.java.net/java/GA/jdk10/10.0.1/fb4372174a714e6b8c52526dc134031e/10/openjdk-10.0.1_linux-x64_bin.tar.gz; fi
+RUN if [ "${DEV_BUILD}" = "true" ]; then tar -xf jdk.tar.gz; fi
+RUN if [ "${DEV_BUILD}" = "true" ]; then rm -f jdk.tar.gz; fi
+
+ARG KERNEL_PATH=./pack/aion.tar.bz2
+# COPY has a different behaviour in docker-compose vs docker so we'll have to use ADD
+# ADD does some magic and automatically unpacks so we need to fix that
+ADD ${KERNEL_PATH} /opt/aion.tar.bz2
+# again different behaivour for ADD when unpacking in docker vs docker-compose
+RUN if [ -d /opt/aion.tar.bz2/aion ]; then mkdir -p /opt/aion; fi
+RUN if [ -d /opt/aion.tar.bz2/aion ]; then mv /opt/aion.tar.bz2/aion/* /opt/aion/; fi
+RUN if [ -d /opt/aion.tar.bz2/aion ]; then rm -rf /opt/aion.tar.bz2; else mv /opt/aion.tar.bz2 /opt/aion; fi
+
+RUN if [ "${DEV_BUILD}" = "true" ]; then cp jdk-10.0.1/lib/libjdwp.so ./aion/rt/lib/.; fi
+RUN if [ "${DEV_BUILD}" = "true" ]; then cp jdk-10.0.1/lib/libdt_socket.so ./aion/rt/lib/.; fi
+RUN if [ "${DEV_BUILD}" = "true" ]; then rm -rf jdk-10.0.1; fi
+
+WORKDIR /opt/aion
+
+COPY ./override-config.py .
+COPY ./create-coinbase.sh .
+COPY ./aion-docker.sh .
+
+CMD [ "/bin/bash", "-c", "./aion-docker.sh" ]

Jenkinsfile

@@ -16,7 +16,7 @@ pipeline {
 
                 sh "git submodule init" 
 
-                sh "git submodule update --recursive --remote --merge"
+                sh "git submodule update --init --recursive"
 
                 sh "${env.ANT_HOME} pack_build"
 
@@ -29,7 +29,7 @@ pipeline {
 
         stage('Archive build output') {
             when {
-                expression { GIT_BRANCH == 'master' || GIT_BRANCH == 'dev' || GIT_BRANCH == 'ci' }
+                expression { GIT_BRANCH == 'master' || GIT_BRANCH == 'dev' || GIT_BRANCH == 'ci' || GIT_BRANCH == 'dev-audit' }
             }
             steps {                
                 archiveArtifacts artifacts: 'pack/aion-v*.tar.bz2'

aion-docker.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+#TODO: Mostly used for development purposes; remove miner setup after changing consensus
+echo "Setting up miner..."
+/bin/bash -c ./create-coinbase.sh
+
+echo "Overriding config parameters..."
+python3 ./override-config.py
+
+#TODO: Mostly used for development purposes
+if [ ! -z "${difficulty}" ]; then
+    echo "Overriding difficulty with ${difficulty}"
+    sed 's/"difficulty": "0x4000"/"difficulty": "'${difficulty}'"/g' -i ./config/genesis.json
+fi
+
+#TODO: We wont's support setting options for the script as the container will have to be teared down after
+#TODO: running and recreated again. This is not feasible in a production environment.Instead of passing options to the
+#TODO: startup script we should provide a rpc/java API that will allow the required functionality to be present after
+#TODO: kernel startup. Until then we can still use that functionality by manually starting a bash session in the
+#TODO: container and running the commands that we need. Check DOCKER.md for more details.
+echo "Starting kernel..."
+/bin/bash -c ./aion.sh

aion.sh

@@ -37,7 +37,16 @@ ARG=$@
 # add execute permission to rt
 chmod +x ./rt/bin/*
 
+# prepare jvm params
+# default to minimum 4gb heap if Xms not set.
+JAVA_OPTS="$JAVA_OPTS"
+if [[ ! ${JAVA_OPTS} = *"-Xms"* ]]; then
+  JAVA_OPTS+=" -Xms4g"
+fi
 
+# to suppress illegal reflective access warning out of xnio and protobuf
+# (we depend on xnio transitively via undertow-core)
+JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED"
 
 ####### WATCHGUARD IMPLEMENTATION #######
 #				    	#
@@ -120,8 +129,8 @@ if $guard; then
 		lastBoot=$newBoot
 
 		# Execute Java kernel
-		env EVMJIT="-cache=1" ./rt/bin/java -Xms4g \
-			-cp "./lib/*:./mod/*" org.aion.Aion "$@" &
+		env EVMJIT="-cache=1" ./rt/bin/java ${JAVA_OPTS} \
+			-cp "./lib/*:./lib/libminiupnp/*:./mod/*" org.aion.Aion "$@" &
 		kPID=$!
 		running=true
 		watching=true
@@ -136,7 +145,6 @@ if $guard; then
 
 		# Watchguard
 		while $watching; do
-
 			sleep $sample
 
 			# [1] Log timestamp (last 60 sec) OR [2] PID process state ZOMBIE/DEAD
@@ -159,7 +167,7 @@ if $guard; then
 			  for ((i=0; i<${#threads[@]}; ++i)); do
 			    tTime=$(ps --pid $kPID | egrep -o "[0-9]{2}\.[0-9]{2} ${threads[i]}" | cut -d" " -f1)
 			    tState=$(./rt/bin/jstack -l $kPID | egrep -A1 "${threads[i]}" | egrep -o "State.*" | cut -d" " -f2)
-			    if [[ $tTime == ${tPrev[i]} ]] && [[ $tState == "BLOCKED" ]]; then 
+			    if [[ $tTime == ${tPrev[i]} ]] && [[ $tState == "BLOCKED" ]]; then
 			      echo "## ${threads[i]} THREAD DEAD ##"
 			      (./rt/bin/jstack -l $kPID | egrep -A1 "${threads[i]}") > threadDump_$countRebounce.txt
 			      watching=false
@@ -195,8 +203,37 @@ if $guard; then
 		echo "############################## REBOUNCE COUNT [$countRebounce] ##############################"
 
 	done
-
 else
-  	env EVMJIT="-cache=1" ./rt/bin/java -Xms4g \
-  		-cp "./lib/*:./mod/*" org.aion.Aion "$@"
+	JAVA_CMD=java
+	if [ -d "./rt" ]; then
+			JAVA_CMD="./rt/bin/java"
+	elif [ -d "./pack/rt" ]; then
+			JAVA_CMD="./pack/rt/bin/java"
+	elif [ -d "$JAVA_HOME" ]; then
+			JAVA_CMD="$JAVA_HOME/bin/java"
+	fi
+
+	# if there's CLI args, we just run it and quit
+	if [ "$#" -gt 0 ]; then
+		env EVMJIT="-cache=1" $JAVA_CMD ${JAVA_OPTS} \
+			-cp "./lib/*:./lib/libminiupnp/*:./mod/*" org.aion.Aion "$@"
+		exit
+	fi
+
+	# otherwise, kernel will start; run it in background
+	trap "exit" INT TERM
+	trap "exit_kernel" EXIT
+
+	exit_kernel() {
+		if [ ! -z "$kernel_pid" ]; then
+			kill "$kernel_pid" &> /dev/null
+			wait "$kernel_pid" &> /dev/null
+		fi
+		exit 1
+	}
+
+  	env EVMJIT="-cache=1" $JAVA_CMD ${JAVA_OPTS} \
+  		-cp "./lib/*:./lib/libminiupnp/*:./mod/*" org.aion.Aion "$@" &
+    kernel_pid=$!
+    wait
 fi