[lib] refs #992. Merge with skycoin/develop.

.gitignore

@@ -121,7 +121,8 @@ src/mesh/TODO
 *.test
 coverage.html
 
-src/gui/integration/wallets/
+src/api/integration/wallets/
+src/api/integration/wallets/
 
 skycoin-cli
 

.travis.yml

@@ -24,6 +24,7 @@ env:
     - BUILD_DIR: build
     - BUILDLIB_DIR: $BUILD_DIR/libskycoin
     - LIB_DIR: lib
+    - BUILD_UI_TRAVIS_DIR: HOME/.skycoin-ui-travis
 
 cache:
   yarn: true
@@ -35,11 +36,16 @@ cache:
   - $HOME/.npm/_prebuilds
   - src/gui/static/node_modules
 
+addons:
+  chrome: stable
+
 install:
   # Install gox
   - go get github.com/gz-c/gox
   # Install dependences for building wallet
   - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PULL_REQUEST" == false ]]; then sudo apt-get install --no-install-recommends -y icnsutils graphicsmagick xz-utils && nvm install 8; fi
+  - mkdir -p "$GOPATH/src/github.com/skycoin" && if [[ ! -d "$GOPATH/src/github.com/skycoin/skycoin" ]] ; then ln -s "$TRAVIS_BUILD_DIR" "$GOPATH/src/github.com/skycoin/skycoin"; fi
+  - cd "$GOPATH/src/github.com/skycoin/skycoin"
   - go get -t ./...
   - make install-linters
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -qq g++-6 && sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-6 90; fi
@@ -49,6 +55,7 @@ install:
 
 before_script:
   - if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == false ]]; then ./ci-scripts/add-key.sh && npm install -g yarn ;fi
+  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then export DISPLAY=:99.0 && sh -e /etc/init.d/xvfb start; fi
 
 script:
   - make lint
@@ -68,13 +75,13 @@ script:
   # - CC=gcc-6 make test-libpy
   # TODO: test ui
   # - make test-ui
+  # - make test-libpy
   - make lint-ui
+  - make build-ui-travis
+  - make test-ui
+  - make test-ui-e2e
   # Build wallets
   - if [[ "$TRAVIS_PULL_REQUEST" == false ]]; then ./ci-scripts/build-wallet.sh; fi
-  # Check that the UI can build. this is done after the wallet build step, because make build-ui
-  # will change the dist/ folder.
-  # If make test-ui can verify that the wallet compiles, we can remove this step after we enable make test-ui
-  - make build-ui
 
 notifications:
   email:

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 
 ### Added
-
+- Add environment variable `DATA_DIR` in CLI's
 - `USE_CSRF` environment variable for CLI, if the remote node has CSRF enabled (CSRF is enabled by default, use `-disable-csrf` to disable)
 - `cli showConfig` command to echo the cli's configuration back to the user
 - Option to generate 12/24 word seed when creating new wallet
@@ -17,21 +17,28 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Add CLI `decryptWallet` command
 - Add CLI `showSeed` command
 - Add `password` argument to the CLI commands of `addPrivateKey`, `createRawTransaction`, `generateAddresses`, `generateWallet`, `send`
+- Support for decoding map values in cipher binary encoder
+- Expose known block height of peer in brand new `height` field added in responses of `/network/connections` API endpoints
+- `-verify-db` option (default true), will verify the database integrity during startup and exit if a problem is found
+- `-reset-corrupt-db` option (default false) will verify the database integrity during startup and reset the db if a problem is found
 
 ### Fixed
 
 - Reduce connection disconnects, improves syncing
 - Fix #1171, update CLI to support wallet encryption
+- Use bolt.Tx correctly for read operations
 
 ### Changed
 
 - JSON 2.0 RPC interface (used by the CLI tool) is now served on the same host interface as the REST API, port `6420`. The additional listener has been removed.
 - CLI's `RPC_ADDR` environment variable must now start with a scheme e.g. `http://127.0.0.1:6420`, previously it did not use a scheme.
+- API response will be gzip compressed if client sends request with 'Accept-Encoding' contains 'gzip' in the header.
 
 ### Removed
 
 - Remove `-rpc-interface-addr`, `-rpc-interface-port` options.  The RPC interface is now on default port `6420` with the REST API.
 - Remove `-rpc-thread-num` option
+- Remove `-connect-to` option
 
 ## [0.23.0] - 2018-04-22
 

Makefile

@@ -11,27 +11,12 @@ GUI_STATIC_DIR = src/gui/static
 # Electron files directory
 ELECTRON_DIR = electron
 
-# ./src folder does not have code
-# ./src/api folder does not have code
-# ./src/util folder does not have code
-# ./src/ciper/* are libraries manually vendored by cipher that do not need coverage
-# ./src/gui/static* are static assets
-# */testdata* folders do not have code
-# ./src/consensus/example has no buildable code
-PACKAGES = $(shell find ./src -type d -not -path '\./src' \
-    							      -not -path '\./src/api' \
-    							      -not -path '\./src/util' \
-    							      -not -path '\./src/consensus/example' \
-    							      -not -path '\./src/gui/static*' \
-    							      -not -path '\./src/cipher/*' \
-    							      -not -path '*/testdata*' \
-    							      -not -path '*/test-fixtures*')
-
 # Compilation output
 BUILD_DIR = build
 BUILDLIB_DIR = $(BUILD_DIR)/libskycoin
 LIB_DIR = lib
 LIB_FILES = $(shell find ./lib/cgo -type f -name "*.go")
+SRC_FILES = $(shell find ./src -type f -name "*.go")
 BIN_DIR = bin
 DOC_DIR = docs
 INCLUDE_DIR = include
@@ -92,7 +77,9 @@ configure-build:
 	mkdir -p $(BUILD_DIR)/usr/tmp $(BUILD_DIR)/usr/lib $(BUILD_DIR)/usr/include
 	mkdir -p $(BUILDLIB_DIR) $(BIN_DIR) $(INCLUDE_DIR)
 
-build-libc: configure-build ## Build libskycoin C client library
+build-libc: configure-build $(BUILDLIB_DIR)/libskycoin.so $(BUILDLIB_DIR)/libskycoin.a ## Build libskycoin C client library
+
+$(BUILDLIB_DIR)/libskycoin.so $(BUILDLIB_DIR)/libskycoin.a: $(LIB_FILES) $(SRC_FILES)
 	rm -Rf $(BUILDLIB_DIR)/*
 	go build -buildmode=c-shared  -o $(BUILDLIB_DIR)/libskycoin.so $(LIB_FILES)
 	go build -buildmode=c-archive -o $(BUILDLIB_DIR)/libskycoin.a  $(LIB_FILES)
@@ -101,11 +88,7 @@ build-libc: configure-build ## Build libskycoin C client library
 ## Build libskycoin C client library and executable C test suites
 ## with debug symbols. Use this target to debug the source code
 ## with the help of an IDE
-build-libc-dbg: configure-build
-	rm -Rf $(BUILDLIB_DIR)/*
-	go build -buildmode=c-shared  -o $(BUILDLIB_DIR)/libskycoin.so $(LIB_FILES)
-	go build -buildmode=c-archive -o $(BUILDLIB_DIR)/libskycoin.a  $(LIB_FILES)
-	mv $(BUILDLIB_DIR)/libskycoin.h $(INCLUDE_DIR)/
+build-libc-dbg: configure-build $(BUILDLIB_DIR)/libskycoin.so $(BUILDLIB_DIR)/libskycoin.a
 	$(CC) -g -o $(BIN_DIR)/test_libskycoin_shared $(LIB_DIR)/cgo/tests/*.c -lskycoin                    $(LDLIBS) $(LDFLAGS)
 	$(CC) -g -o $(BIN_DIR)/test_libskycoin_static $(LIB_DIR)/cgo/tests/*.c $(BUILDLIB_DIR)/libskycoin.a $(LDLIBS) $(LDFLAGS)
 
@@ -124,15 +107,17 @@ docs: docs-libc
 
 lint: ## Run linters. Use make install-linters first.
 	vendorcheck ./...
-	gometalinter --deadline=3m --concurrency=2 --disable-all --tests --vendor --skip=lib/cgo \
+	gometalinter --deadline=3m --concurrency=2 --disable-all --tests --vendor --skip=lib/cgo --warn-unmatched-nolint \
 		-E goimports \
 		-E golint \
 		-E varcheck \
+		-E unparam \
 		./...
 	# lib cgo can't use golint because it needs export directives in function docstrings that do not obey golint rules
-	gometalinter --deadline=3m --concurrency=2 --disable-all --tests --vendor \
+	gometalinter --deadline=3m --concurrency=2 --disable-all --tests --vendor --warn-unmatched-nolint \
 		-E goimports \
 		-E varcheck \
+		-E unparam \
 		./lib/cgo/...
 
 check: lint test integration-test-stable integration-test-stable-disable-csrf integration-test-disable-wallet-api integration-test-disable-seed-api ## Run tests and linters
@@ -187,11 +172,16 @@ lint-ui:  ## Lint the UI code
 
 test-ui:  ## Run UI tests
 	cd $(GUI_STATIC_DIR) && npm run test
-	cd $(GUI_STATIC_DIR) && npm run e2e
+
+test-ui-e2e:  ## Run UI e2e tests
+	./ci-scripts/ui-e2e.sh
 
 build-ui:  ## Builds the UI
 	cd $(GUI_STATIC_DIR) && npm run build
 
+build-ui-travis:  ## Builds the UI for travis
+	cd $(GUI_STATIC_DIR) && npm run build-travis
+
 release: ## Build electron apps, the builds are located in electron/release folder.
 	cd $(ELECTRON_DIR) && ./build.sh
 	@echo release files are in the folder of electron/release

ci-scripts/integration-test-disable-seed-api.sh

@@ -55,7 +55,7 @@ echo "starting skycoin node in background with http listener on $HOST"
 ./skycoin-integration -disable-networking=true \
                       -web-interface-port=$PORT \
                       -download-peerlist=false \
-                      -db-path=./src/gui/integration/test-fixtures/blockchain-180.db \
+                      -db-path=./src/api/integration/testdata/blockchain-180.db \
                       -db-read-only=true \
                       -rpc-interface=true \
                       -launch-browser=false \
@@ -76,7 +76,7 @@ set +e
 if [[ -z $TEST || $TEST = "gui" ]]; then
 
 SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE SKYCOIN_NODE_HOST=$HOST WALLET_DIR=$WALLET_DIR \
-    go test ./src/gui/integration/... -timeout=30s $VERBOSE $RUN_TESTS
+    go test ./src/api/integration/... -timeout=30s $VERBOSE $RUN_TESTS
 
 GUI_FAIL=$?
 
@@ -85,7 +85,7 @@ fi
 if [[ -z $TEST  || $TEST = "cli" ]]; then
 
 # SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE RPC_ADDR=$RPC_ADDR \
-#     go test ./src/api/cli/integration/... -timeout=30s $VERBOSE $RUN_TESTS
+#     go test ./src/cli/integration/... -timeout=30s $VERBOSE $RUN_TESTS
 
 CLI_FAIL=$?
 

ci-scripts/integration-test-disable-wallet-api.sh

@@ -11,7 +11,6 @@ HOST="http://127.0.0.1:$PORT"
 RPC_ADDR="http://127.0.0.1:$RPC_PORT"
 MODE="disable-wallet-api"
 BINARY="skycoin-integration"
-TEST=""
 UPDATE=""
 # run go test with -v flag
 VERBOSE=""
@@ -26,7 +25,6 @@ GOLDFLAGS="-X main.Commit=${COMMIT} -X main.Branch=${BRANCH}"
 usage () {
   echo "Usage: $SCRIPT"
   echo "Optional command line arguments"
-  echo "-t <string>  -- Test to run, gui or cli; empty runs both tests"
   echo "-r <string>  -- Run test with -run flag"
   echo "-u <boolean> -- Update stable testdata"
   echo "-v <boolean> -- Run test with -v flag"
@@ -39,7 +37,6 @@ while getopts "h?t:r:uvf" args; do
     h|\?)
         usage;
         exit;;
-    t ) TEST=${OPTARG};;
     r ) RUN_TESTS="-run ${OPTARG}";;
     u ) UPDATE="--update";;
     v ) VERBOSE="-v";;
@@ -68,7 +65,7 @@ echo "starting skycoin node in background with http listener on $HOST"
 ./skycoin-integration -disable-networking=true \
                       -web-interface-port=$PORT \
                       -download-peerlist=false \
-                      -db-path=./src/gui/integration/test-fixtures/blockchain-180.db \
+                      -db-path=./src/api/integration/testdata/blockchain-180.db \
                       -db-read-only=true \
                       -rpc-interface=true \
                       -launch-browser=false \
@@ -85,24 +82,10 @@ echo "done sleeping"
 
 set +e
 
-if [[ -z $TEST || $TEST = "gui" ]]; then
-
 SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE SKYCOIN_NODE_HOST=$HOST WALLET_DIR=$WALLET_DIR \
-    go test ./src/gui/integration/... $FAILFAST $UPDATE -timeout=30s $VERBOSE $RUN_TESTS
-
-GUI_FAIL=$?
-
-fi
-
-if [[ -z $TEST  || $TEST = "cli" ]]; then
-
-SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE RPC_ADDR=$RPC_ADDR \
-    go test ./src/api/cli/integration/... $FAILFAST $UPDATE -timeout=30s $VERBOSE $RUN_TESTS
-
-CLI_FAIL=$?
-
-fi
+    go test ./src/api/integration/... $FAILFAST $UPDATE -timeout=30s $VERBOSE $RUN_TESTS
 
+FAIL=$?
 
 echo "shutting down skycoin node"
 
@@ -112,12 +95,4 @@ wait $SKYCOIN_PID
 
 rm "$BINARY"
 
-
-if [[ (-z $TEST || $TEST = "gui") && $GUI_FAIL -ne 0 ]]; then
-  exit $GUI_FAIL
-elif [[ (-z $TEST || $TEST = "cli") && $CLI_FAIL -ne 0 ]]; then
-  exit $CLI_FAIL
-else
-  exit 0
-fi
-# exit $FAIL
+exit $FAIL

ci-scripts/integration-test-live.sh

@@ -57,7 +57,9 @@ set -euxo pipefail
 
 echo "checking if skycoin node is running"
 
-http_proxy="" https_proxy="" wget -O- $HOST 2>&1 >/dev/null
+HEALTH="$HOST/health"
+
+http_proxy="" https_proxy="" wget -O- $HEALTH 2>&1 >/dev/null
 
 if [ ! $? -eq 0 ]; then
     echo "Skycoin node is not running on $HOST"
@@ -67,13 +69,13 @@ fi
 if [[ -z $TEST || $TEST = "gui" ]]; then
 
 SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE SKYCOIN_NODE_HOST=$HOST \
-    go test ./src/gui/integration/... $FAILFAST $UPDATE -timeout=$TIMEOUT $VERBOSE $RUN_TESTS $TEST_LIVE_WALLET
+    go test ./src/api/integration/... $FAILFAST $UPDATE -timeout=$TIMEOUT $VERBOSE $RUN_TESTS $TEST_LIVE_WALLET
 
 fi
 
 if [[ -z $TEST || $TEST = "cli" ]]; then
 
 SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE RPC_ADDR=$RPC_ADDR SKYCOIN_NODE_HOST=$HOST USE_CSRF=$USE_CSRF \
-    go test ./src/api/cli/integration/... $FAILFAST $UPDATE -timeout=$TIMEOUT $VERBOSE $RUN_TESTS $TEST_LIVE_WALLET
+    go test ./src/cli/integration/... $FAILFAST $UPDATE -timeout=$TIMEOUT $VERBOSE $RUN_TESTS $TEST_LIVE_WALLET
 
 fi

ci-scripts/integration-test-stable.sh

@@ -70,7 +70,7 @@ echo "starting skycoin node in background with http listener on $HOST"
 ./skycoin-integration -disable-networking=true \
                       -web-interface-port=$PORT \
                       -download-peerlist=false \
-                      -db-path=./src/gui/integration/test-fixtures/blockchain-180.db \
+                      -db-path=./src/api/integration/testdata/blockchain-180.db \
                       -db-read-only=true \
                       -rpc-interface=true \
                       -launch-browser=false \
@@ -92,7 +92,7 @@ set +e
 if [[ -z $TEST || $TEST = "gui" ]]; then
 
 SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE SKYCOIN_NODE_HOST=$HOST \
-    go test ./src/gui/integration/... $UPDATE -timeout=3m $VERBOSE $RUN_TESTS
+    go test ./src/api/integration/... $UPDATE -timeout=3m $VERBOSE $RUN_TESTS
 
 GUI_FAIL=$?
 
@@ -101,7 +101,7 @@ fi
 if [[ -z $TEST  || $TEST = "cli" ]]; then
 
 SKYCOIN_INTEGRATION_TESTS=1 SKYCOIN_INTEGRATION_TEST_MODE=$MODE RPC_ADDR=$RPC_ADDR USE_CSRF=$USE_CSRF \
-    go test ./src/api/cli/integration/... $UPDATE -timeout=3m $VERBOSE $RUN_TESTS
+    go test ./src/cli/integration/... $UPDATE -timeout=3m $VERBOSE $RUN_TESTS
 
 CLI_FAIL=$?
 

lib/cgo/api.cli.cli.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	cli "github.com/skycoin/skycoin/src/api/cli"
+	cli "github.com/skycoin/skycoin/src/cli"
 )
 
 /*

lib/cgo/api.cli.create_rawtx.go

@@ -12,7 +12,7 @@ import "C"
 import (
 	"unsafe"
 
-	"github.com/skycoin/skycoin/src/api/cli"
+	"github.com/skycoin/skycoin/src/cli"
 	//	"github.com/skycoin/skycoin/src/wallet"
 )
 

lib/cgo/libsky_handle.go

@@ -12,7 +12,7 @@ import "C"
 import (
 	"unsafe"
 
-	cli "github.com/skycoin/skycoin/src/api/cli"
+	cli "github.com/skycoin/skycoin/src/cli"
 	webrpc "github.com/skycoin/skycoin/src/api/webrpc"
 )
 

run-with-rpc.sh

@@ -1,19 +1,3 @@
 #!/usr/bin/env bash
 
-set -x
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-echo "skycoin binary dir:" "$DIR"
-pushd "$DIR" >/dev/null
-
-COMMIT=$(git rev-parse HEAD)
-BRANCH=$(git rev-parse --abbrev-ref HEAD)
-GOLDFLAGS="-X main.Commit=${COMMIT} -X main.Branch=${BRANCH}"
-
-go run -ldflags "${GOLDFLAGS}" cmd/skycoin/skycoin.go \
-    -gui-dir="${DIR}/src/gui/static/" \
-    -launch-browser=true \
-    -enable-wallet-api=true \
-    $@
-
-popd >/dev/null
+./run.sh -rpc-interface=true