Merge bitcoin#17725: ci: Add valgrind run

facb416 ci: Add valgrind run (MarcoFalke)

Pull request description:

  Fixes bitcoin#17460

ACKs for top commit:
  practicalswift:
    ACK facb416

Tree-SHA512: 55396e548a76f976d7b7170b68bc5f93cfd44656162267172f66db7eb549699a2a22d3b1bb0d5f180fe0697931939e652c8cdb86b435e81e7ce572485798009d

.gitlab-ci.yml

@@ -279,6 +279,15 @@ linux64_nowallet-build:
   variables:
     BUILD_TARGET: linux64_nowallet
 
+linux64_valgrind-build:
+  extends:
+    - .build-template
+    - .skip-in-fast-mode-template
+  needs:
+    - x86_64-pc-linux-gnu-debug
+  variables:
+    BUILD_TARGET: linux64_valgrind
+
 mac-build:
   extends:
     - .build-template
@@ -323,3 +332,12 @@ linux64_ubsan-test:
     - linux64_ubsan-build
   variables:
     BUILD_TARGET: linux64_ubsan
+
+linux64_valgrind-test:
+  extends:
+    - .test-template
+    - .skip-in-fast-mode-template
+  needs:
+    - linux64_valgrind-build
+  variables:
+    BUILD_TARGET: linux64_valgrind

.travis.yml

@@ -225,7 +225,7 @@ after_success:
         FILE_ENV="./ci/test/00_setup_env_win64.sh"
 
     - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [focal]  [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]'
+      name: 'x86_64 Linux  [GOAL: install]  [focal]  [uses qt5 dev package and some depends packages] [unsigned char]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_qt5.sh"
 # x86_64 Linux (xenial, no depends, only system libs, sanitizers: thread (TSan))
@@ -240,6 +240,11 @@ after_success:
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_asan.sh"
 
+    - stage: test
+      name: 'x86_64 Linux  [GOAL: install]  [focal]  [no depends, only system libs, valgrind]'
+      env: >-
+        FILE_ENV="./ci/test/00_setup_env_native_valgrind.sh"
+
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [focal]  [no wallet]'
       env: >-

ci/dash/build_src.sh

@@ -56,6 +56,11 @@ cd dashcore-$BUILD_TARGET
 
 make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
 
+if [ -n "$USE_VALGRIND" ]; then
+    echo "valgrind in USE!"
+    ${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh
+fi
+
 if [ "$RUN_SYMBOL_TESTS" = "true" ]; then
   make $MAKEJOBS -C src check-symbols
 fi

ci/dash/matrix.sh

@@ -32,6 +32,8 @@ elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then
   source ./ci/test/00_setup_env_native_sqlite.sh
 elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then
   source ./ci/test/00_setup_env_native_nowallet.sh
+elif [ "$BUILD_TARGET" = "linux64_valgrind" ]; then
+  source ./ci/test/00_setup_env_native_valgrind.sh
 elif [ "$BUILD_TARGET" = "mac" ]; then
   source ./ci/test/00_setup_env_mac.sh
 elif [ "$BUILD_TARGET" = "s390x" ]; then

ci/test/00_setup_env_native_valgrind.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev"
+export USE_VALGRIND=1
+export NO_DEPENDS=1
+export TEST_RUNNER_EXTRA="p2p_instantsend.py"  # Only run one test for now. TODO enable all and bump timeouts
+export RUN_FUNCTIONAL_TESTS=true
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++"  # TODO enable GUI

ci/test/04_install.sh

@@ -47,7 +47,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=
 export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
-env | grep -E '^(QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
+env | grep -E '^(BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"
 elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)

ci/test/wrap-valgrind.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in "${BASE_OUTDIR}/bin"/*; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
+      echo "Wrap $b ..."
+      mv "$b" "${b}_orig"
+      echo '#!/usr/bin/env bash' > "$b"
+      echo "valgrind --gen-suppressions=all --quiet --error-exitcode=1 --suppressions=${BASE_ROOT_DIR}/contrib/valgrind.supp \"${b}_orig\" \"\$@\"" >> "$b"
+      chmod +x "$b"
+    done
+done

contrib/containers/ci/Dockerfile

@@ -74,6 +74,7 @@ RUN apt-get update && apt-get install $APT_ARGS \
     nsis \
     python3-zmq \
     parallel \
+    valgrind \
     wine-stable \
     wine32 \
     wine64 \

contrib/valgrind.supp

@@ -30,17 +30,58 @@
    Memcheck:Cond
    obj:*/libdb_cxx-*.so
    fun:__log_put
-   obj:*/libdb_cxx-*.so
-   fun:__log_put_record
 }
 {
    Suppress libdb warning
    Memcheck:Param
    pwrite64(buf)
    fun:pwrite
    fun:__os_io
+}
+{
+   Suppress libdb warning
+   Memcheck:Cond
+   fun:__log_putr.isra.1
+}
+{
+   Suppress libdb warning
+   Memcheck:Param
+   pwrite64(buf)
+   fun:pwrite
+   fun:__os_io
+   obj:*/libdb_cxx-*.so
+}
+{
+   Suppress uninitialized bytes warning in compat code
+   Memcheck:Param
+   ioctl(TCSET{S,SW,SF})
+   fun:tcsetattr
+}
+{
+   Suppress libdb warning
+   Memcheck:Leak
+   fun:malloc
+   ...
    obj:*/libdb_cxx-*.so
 }
+{
+   Suppress leaks on init
+   Memcheck:Leak
+   ...
+   fun:_Z11AppInitMainR11NodeContext
+}
+{
+   Suppress leaks on shutdown
+   Memcheck:Leak
+   ...
+   fun:_Z8ShutdownR11NodeContext
+}
+{
+   Ignore GUI warning
+   Memcheck:Leak
+   ...
+   obj:/usr/lib64/libgdk-3.so.0.2404.7
+}
 {
    Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113
    Memcheck:Leak
@@ -56,24 +97,57 @@
    ...
    fun:_ZN7leveldbL14InitDefaultEnvEv
 }
+{
+   Suppress leveldb leak
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   ...
+   fun:_ZN7leveldb6DBImpl14BackgroundCallEv
+}
+{
+   Suppress leveldb leak
+   Memcheck:Leak
+   fun:_Znwm
+   ...
+   fun:GetCoin
+}
 {
    Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626)
    Memcheck:Addr16
    fun:__wcsnlen_sse4_1
    fun:wcsnrtombs
 }
+{
+   Suppress wcsnrtombs warning (remove after removing boost::fs)
+   Memcheck:Cond
+   ...
+   fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE
+   fun:unique_path
+}
+{
+   Suppress boost warning
+   Memcheck:Leak
+   fun:_Znwm
+   ...
+   fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE
+   fun:_ZN5boost9unit_test9framework3runEmb
+   fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc
+   fun:main
+}
 {
    Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9)
    Memcheck:Cond
    fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE
-   fun:_ZN5boost10filesystem6detail28directory_iterator_constructERNS0_18directory_iteratorERKNS0_4pathEPNS_6system10error_codeE
+   ...
    obj:*/libboost_filesystem.so.*
 }
 {
    Suppress boost::filesystem warning (could be related: https://stackoverflow.com/questions/9830182/function-boostfilesystemcomplete-being-reported-as-possible-memory-leak-by-v)
    Memcheck:Leak
    match-leak-kinds: reachable
    fun:_Znwm
+   ...
    fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_
 }
 {