Catch up with Core until commit f617e05c38c9c7e663b824fefe5329cc5cbbf26a

.appveyor.yml

@@ -0,0 +1,58 @@
+version: '{branch}.{build}'
+skip_tags: true
+image: Visual Studio 2017
+configuration: Release
+platform: x64
+clone_depth: 5
+environment:
+  APPVEYOR_SAVE_CACHE_ON_ERROR: true
+  CLCACHE_SERVER: 1
+  PACKAGES: boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb secp256k1 leveldb
+  PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%'
+  PYTHONUTF8: 1
+cache:
+- C:\tools\vcpkg\installed
+- C:\Users\appveyor\clcache
+install:
+- cmd: pip install --quiet git+https://github.com/frerich/[email protected]
+# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
+# - cmd: pip install zmq
+- cmd: vcpkg install --triplet %PLATFORM%-windows-static %PACKAGES% > NUL
+- cmd: del /s /q C:\Tools\vcpkg\installed\%PLATFORM%-windows-static\debug # Remove unused debug library
+before_build:
+- ps:  clcache -M 536870912
+- cmd: python build_msvc\msvc-autogen.py
+- ps:  $files = (Get-ChildItem -Recurse | where {$_.extension -eq ".vcxproj"}).FullName
+- ps:  for (${i} = 0; ${i} -lt ${files}.length; ${i}++) {
+           ${content} = (Get-Content ${files}[${i}]);
+           ${content} = ${content}.Replace("</RuntimeLibrary>", "</RuntimeLibrary><DebugInformationFormat>None</DebugInformationFormat>");
+           ${content} = ${content}.Replace("<WholeProgramOptimization>true", "<WholeProgramOptimization>false");
+           ${content} = ${content}.Replace("NDEBUG;", "");
+           Set-Content ${files}[${i}] ${content};
+       }
+- ps:  Start-Process clcache-server
+- ps:  fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache)
+build_script:
+- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:q /nowarn:C4244;C4267;C4715 /nologo
+after_build:
+- ps:  fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance)
+- ps:  clcache -z
+before_test:
+- ps:  ${conf_ini} = (Get-Content([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini.in")))
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@abs_top_srcdir@", ${env:APPVEYOR_BUILD_FOLDER})
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@abs_top_builddir@", ${env:APPVEYOR_BUILD_FOLDER})
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@EXEEXT@", ".exe")
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@ENABLE_WALLET_TRUE@", "")
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@BUILD_BITCOIN_CLI_TRUE@", "")
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@BUILD_BITCOIND_TRUE@", "")
+- ps:  ${conf_ini} = ${conf_ini}.Replace("@ENABLE_ZMQ_TRUE@", "")
+- ps:  ${utf8} = New-Object System.Text.UTF8Encoding ${false}
+- ps:  '[IO.File]::WriteAllLines([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini"), ${conf_ini}, ${utf8})'
+- ps:  move "build_msvc\${env:PLATFORM}\${env:CONFIGURATION}\*.exe" src
+test_script:
+- cmd: src\test_bitcoin.exe -k stdout -e stdout 2> NUL
+- cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL
+- ps:  python test\util\bitcoin-util-test.py
+- cmd: python test\util\rpcauth-test.py
+- cmd: python test\functional\test_runner.py --ci --force --quiet --combinedlogslen=4000 --failfast
+deploy: off

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,31 @@
+Pull requests without a rationale and clear improvement may be closed
+immediately.
+
+Please provide clear motivation for your patch and explain how it improves
+Bitcoin Core user experience or Bitcoin Core developer experience
+significantly.
+
+* Any test improvements or new tests that improve coverage are always welcome.
+* All other changes should have accompanying unit tests (see `src/test/`) or
+  functional tests (see `test/`). Contributors should note which tests cover
+  modified code. If no tests exist for a region of modified code, new tests
+  should accompany the change.
+* Bug fixes are most welcome when they come with steps to reproduce or an
+  explanation of the potential issue as well as reasoning for the way the bug
+  was fixed.
+* Features are welcome, but might be rejected due to design or scope issues.
+  If a feature is based on a lot of dependencies, contributors should first
+  consider building the system outside of Bitcoin Core, if possible.
+* Refactoring changes are only accepted if they are required for a feature or
+  bug fix or otherwise improve developer experience significantly. For example,
+  most "code style" refactoring changes require a thorough explanation why they
+  are useful, what downsides they have and why they *significantly* improve
+  developer experience or avoid serious programming bugs. Note that code style
+  is often a subjective matter. Unless they are explicitly mentioned to be
+  preferred in the [developer notes](/doc/developer-notes.md), stylistic code
+  changes are usually rejected.
+
+Bitcoin Core has a thorough review process and even the most trivial change
+needs to pass a lot of eyes and requires non-zero or even substantial time
+effort to review. There is a huge lack of active reviewers on the project, so
+patches often sit for a long time.

.gitignore

@@ -116,3 +116,6 @@ test/cache/*
 
 libbitcoinconsensus.pc
 contrib/devtools/split-debug.sh
+
+# Output from running db4 installation
+db4/

.travis.yml

@@ -16,7 +16,6 @@ env:
     - MAKEJOBS=-j3
     - RUN_UNIT_TESTS=true
     - RUN_FUNCTIONAL_TESTS=true
-    - RUN_BENCH=false  # Set to true for any one job that has debug enabled, to quickly check bench is not crashing or hitting assertions
     - DOCKER_NAME_TAG=ubuntu:18.04
     - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
     - CCACHE_SIZE=100M
@@ -35,7 +34,7 @@ install:
 before_script:
   - set -o errexit; source .travis/test_05_before_script.sh
 script:
-  - set -o errexit; source .travis/test_06_script.sh
+  - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script.sh; fi
 after_script:
   - echo $TRAVIS_COMMIT_RANGE
   - echo $TRAVIS_COMMIT_LOG
@@ -58,8 +57,7 @@ jobs:
     - stage: test
       env: >-
         HOST=arm-linux-gnueabihf
-        PACKAGES="g++-arm-linux-gnueabihf"
-        DEP_OPTS="NO_QT=1"
+        PACKAGES="python3 g++-arm-linux-gnueabihf"
         RUN_UNIT_TESTS=false
         RUN_FUNCTIONAL_TESTS=false
         GOAL="install"
@@ -71,25 +69,22 @@ jobs:
       env: >-
         HOST=i686-w64-mingw32
         DPKG_ADD_ARCH="i386"
-        DEP_OPTS="NO_QT=1"
         PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32"
         GOAL="install"
-        BITCOIN_CONFIG="--enable-reduce-exports"
+        BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
         RUN_UNIT_TESTS=false # secp-zkp times out Travis, get 32-bit build in upstream?
 # Win64
     - stage: test
       env: >-
         HOST=x86_64-w64-mingw32
-        DEP_OPTS="NO_QT=1"
         PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-reduce-exports"
+        GOAL="deploy"
+        BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
 # 32-bit + dash
     - stage: test
       env: >-
         HOST=i686-pc-linux-gnu
         PACKAGES="g++-multilib python3-zmq"
-        DEP_OPTS="NO_QT=1"
         GOAL="install"
         BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
         CONFIG_SHELL="/bin/dash"
@@ -112,24 +107,23 @@ jobs:
         GOAL="install"
         BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
         RUN_UNIT_TESTS=false
-# x86_64 Linux (no depends, only system libs)
+# x86_64 Linux (xenial, no depends, only system libs)
     - stage: test
       env: >-
         HOST=x86_64-unknown-linux-gnu
-        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
+        DOCKER_NAME_TAG=ubuntu:16.04
+        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
         NO_DEPENDS=1
         GOAL="install"
         BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER"
-# x86_64 Linux (sanitizers)
+# x86_64 Linux (no depends, only system libs, sanitizers: undefined (UBSAN) + integer)
     - stage: test
       env: >-
         HOST=x86_64-unknown-linux-gnu
-        PACKAGES="clang python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
+        PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
         NO_DEPENDS=1
-        RUN_BENCH=true
-        RUN_FUNCTIONAL_TESTS=false # Disabled for now, can be combined with the other x86_64 linux NO_DEPENDS job when functional tests pass the sanitizers
         GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=undefined CC=clang CXX=clang++"
+        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=integer,undefined CC=clang CXX=clang++"
 # x86_64 Linux, No wallet, no QT(build timing out with for some reason)
     - stage: test
       env: >-
@@ -146,7 +140,7 @@ jobs:
         OSX_SDK=10.11
         RUN_UNIT_TESTS=false
         RUN_FUNCTIONAL_TESTS=false
-        GOAL="all deploy"
+        GOAL="deploy"
         BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
 
 # x86_64 Linux w/ Bitcoin functional tests (Qt5 & system libs)

.travis/lint_04_install.sh

@@ -8,3 +8,4 @@ export LC_ALL=C
 
 travis_retry pip install codespell==1.13.0
 travis_retry pip install flake8==3.5.0
+travis_retry pip install vulture==0.29

.travis/lint_06_script.sh

@@ -19,6 +19,7 @@ test/lint/check-rpc-mappings.py .
 test/lint/lint-all.sh
 
 if [ "$TRAVIS_REPO_SLUG" = "ElementsProject/elements" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then
+    git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit
     while read -r LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys &&
-    travis_wait 50 contrib/verify-commits/verify-commits.py;
+    travis_wait 50 contrib/verify-commits/verify-commits.py --clean-merge=2;
 fi

.travis/test_03_before_install.sh

@@ -7,6 +7,8 @@
 export LC_ALL=C.UTF-8
 
 PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
+# Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers.
+PATH=$PATH:/usr/lib/llvm-6.0/bin/
 export PATH
 
 BEGIN_FOLD () {

.travis/test_04_install.sh

@@ -7,7 +7,8 @@
 export LC_ALL=C.UTF-8
 
 travis_retry docker pull "$DOCKER_NAME_TAG"
-env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL)' | tee /tmp/env
+export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/contrib/sanitizers-ubsan.suppressions:print_stacktrace=1:halt_on_error=1"
+env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|UBSAN_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"
 fi

.travis/test_06_script.sh

@@ -50,19 +50,13 @@ if [ "$RUN_UNIT_TESTS" = "true" ]; then
   END_FOLD
 fi
 
-if [ "$RUN_BENCH" = "true" ]; then
-  BEGIN_FOLD bench
-  DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib $OUTDIR/bin/bench_bitcoin -scaling=0.001
-  END_FOLD
-fi
-
 if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
   extended="--extended --exclude feature_pruning"
 fi
 
 if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
   BEGIN_FOLD functional-tests
-  DOCKER_EXEC test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet --failfast ${extended}
+  DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast ${extended}
   END_FOLD
 fi
 

CONTRIBUTING.md

@@ -67,6 +67,8 @@ If a particular commit references another issue, please add the reference. For
 example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords
 will cause the corresponding issue to be closed when the pull request is merged.
 
+Commit messages should never contain any `@` mentions.
+
 Please refer to the [Git manual](https://git-scm.com/doc) for more information
 about Git.
 
@@ -135,6 +137,10 @@ before it will be merged. The basic squashing workflow is shown below.
     # Save and quit.
     git push -f # (force push to GitHub)
 
+Please update the resulting commit message if needed, it should read as a
+coherent message. In most cases this means that you should not just list the
+interim commits.
+
 If you have problems with squashing (or other workflows with `git`), you can
 alternatively enable "Allow edits from maintainers" in the right GitHub
 sidebar and ask for help in the pull request.

Makefile.am

@@ -171,7 +171,7 @@ $(BITCOIN_CLI_BIN): FORCE
 	$(MAKE) -C src $(@F)
 
 if USE_LCOV
-LCOV_FILTER_PATTERN=-p "/usr/include/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1"
+LCOV_FILTER_PATTERN=-p "/usr/include/" -p "/usr/lib/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1"
 
 baseline.info:
 	$(LCOV) -c -i -d $(abs_builddir)/src -o $@
@@ -299,5 +299,5 @@ clean-docs:
 
 clean-local: clean-docs
 	rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP)
-	rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache
+	rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__
 

build_msvc/common.vcxproj

@@ -12,4 +12,9 @@
           Outputs="$(MSBuildThisFileDirectory)..\src\config\bitcoin-config.h">
       <Copy SourceFiles="$(MSBuildThisFileDirectory)bitcoin_config.h" DestinationFiles="$(MSBuildThisFileDirectory)..\src\config\bitcoin-config.h" />
   </Target>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions>
+    </ClCompile>
+  </ItemDefinitionGroup>
 </Project>

build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj

@@ -40,7 +40,7 @@
     <ClCompile Include="..\..\src\script\script.cpp" />
     <ClCompile Include="..\..\src\script\script_error.cpp" />
     <ClCompile Include="..\..\src\uint256.cpp" />
-    <ClCompile Include="..\..\src\utilstrencodings.cpp" />
+    <ClCompile Include="..\..\src\util\strencodings.cpp" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

build_msvc/msvc-autogen.py

@@ -16,10 +16,6 @@
 ]
 
 ignore_list = [
-    'rpc/net.cpp',
-    'interfaces/handler.cpp',
-    'interfaces/node.cpp',
-    'interfaces/wallet.cpp',
 ]
 
 lib_sources = {}
@@ -32,7 +28,9 @@ def parse_makefile(makefile):
             if current_lib:
                 source = line.split()[0]
                 if source.endswith('.cpp') and not source.startswith('$') and source not in ignore_list:
-                    lib_sources[current_lib].append(source.replace('/', '\\'))
+                    source_filename = source.replace('/', '\\')
+                    object_filename = source.replace('/', '_')[:-4] + ".obj"
+                    lib_sources[current_lib].append((source_filename, object_filename))
                 if not line.endswith('\\'):
                     current_lib = ''
                 continue
@@ -51,8 +49,10 @@ def main():
     for key, value in lib_sources.items():
         vcxproj_filename = os.path.abspath(os.path.join(os.path.dirname(__file__), key, key + '.vcxproj'))
         content = ''
-        for source_filename in value:
-            content += '    <ClCompile Include="..\\..\\src\\' + source_filename + '" />\n'
+        for source_filename, object_filename in value:
+            content += '    <ClCompile Include="..\\..\\src\\' + source_filename + '">\n'
+            content += '      <ObjectFileName>$(IntDir)' + object_filename + '</ObjectFileName>\n'
+            content += '    </ClCompile>\n'
         with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file:
             with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file:
                 vcxproj_file.write(vcxproj_in_file.read().replace(

build_msvc/test_bitcoin/test_bitcoin.vcxproj

@@ -24,7 +24,7 @@
     <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" />
     <ClCompile Include="..\..\src\test\test_bitcoin.cpp" />
     <ClCompile Include="..\..\src\test\test_bitcoin_main.cpp" />
-    <ClCompile Include="..\..\src\wallet\test\wallet_test_fixture.cpp" />
+    <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">