Merge pull request #2368 from div72/rand

random: port upstream random changes

contrib/devtools/test_deterministic_coverage.sh

@@ -13,28 +13,9 @@ export LC_ALL=C
 GCOV_EXECUTABLE="gcov"
 
 # Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism.
-NON_DETERMINISTIC_TESTS=(
-    "blockfilter_index_tests/blockfilter_index_initial_sync"  # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... }
-    "coinselector_tests/knapsack_solver_test"                 # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2))
-    "fs_tests/fsbridge_fstream"                               # deterministic test failure?
-    "miner_tests/CreateNewBlock_validity"                     # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "scheduler_tests/manythreads"                             # scheduler.cpp: CScheduler::serviceQueue()
-    "scheduler_tests/singlethreadedscheduler_ordered"         # scheduler.cpp: CScheduler::serviceQueue()
-    "txvalidationcache_tests/checkinputs_test"                # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "txvalidationcache_tests/tx_mempool_block_doublespend"    # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "txindex_tests/txindex_initial_sync"                      # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "txvalidation_tests/tx_mempool_reject_coinbase"           # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/coin_mark_dirty_immature_credit"            # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/dummy_input_size_test"                      # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/importmulti_rescan"                         # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/importwallet_rescan"                        # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/ListCoins"                                  # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/scan_for_wallet_transactions"               # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "wallet_tests/wallet_disableprivkeys"                     # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-)
+NON_DETERMINISTIC_TESTS=()
 
-TEST_BITCOIN_BINARY="src/test/test_bitcoin"
+TEST_BITCOIN_BINARY="src/test/test_gridcoin"
 
 print_usage() {
     echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]"

src/Makefile.am

@@ -156,6 +156,8 @@ GRIDCOIN_CORE_H = \
     prevector.h \
     primitives/transaction.h \
     protocol.h \
+    random.h \
+    randomenv.h \
     reverselock.h \
     rpc/blockchain.h \
     rpc/client.h \
@@ -257,6 +259,8 @@ GRIDCOIN_CORE_CPP = addrdb.cpp \
     policy/policy.cpp \
     primitives/transaction.cpp \
     protocol.cpp \
+    random.cpp \
+    randomenv.cpp \
     rpc/blockchain.cpp \
     rpc/client.cpp \
     rpc/dataacq.cpp \

src/Makefile.test.include

@@ -56,6 +56,7 @@ GRIDCOIN_TESTS =\
 	test/mruset_tests.cpp \
 	test/multisig_tests.cpp \
 	test/netbase_tests.cpp \
+	test/random_tests.cpp \
 	test/rpc_tests.cpp \
 	test/sanity_tests.cpp \
 	test/scheduler_tests.cpp \

src/addrdb.cpp

@@ -9,7 +9,7 @@
 #include <chainparams.h>
 #include <clientversion.h>
 #include <hash.h>
-// #include <random.h>
+#include <random.h>
 // #include <streams.h>
 #include <tinyformat.h>
 // #include <util/system.h>
@@ -41,7 +41,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data
 {
     // Generate random temporary filename
     unsigned short randv = 0;
-    RAND_bytes((unsigned char*)&randv, sizeof(randv));
+    GetRandBytes((unsigned char*)&randv, sizeof(randv));
     std::string tmpfn = strprintf("%s.%04x", prefix, randv);
 
     // open temp output file, and associate with CAutoFile

src/addrman.cpp

@@ -3,6 +3,7 @@
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
 #include "addrman.h"
+#include "random.h"
 #include "streams.h"
 
 using namespace std;

src/addrman.h

@@ -7,6 +7,7 @@
 
 #include "netbase.h"
 #include "protocol.h"
+#include "random.h"
 #include "util.h"
 #include "sync.h"
 
@@ -383,7 +384,7 @@ class CAddrMan
     CAddrMan() : vRandom(0), vvTried(ADDRMAN_TRIED_BUCKET_COUNT, std::vector<int>(0)), vvNew(ADDRMAN_NEW_BUCKET_COUNT, std::set<int>())
     {
          nKey.resize(32);
-         RAND_bytes(&nKey[0], 32);
+         GetRandBytes(nKey.data(), 32);
 
          nIdCount = 0;
          nTried = 0;
@@ -504,7 +505,7 @@ class CAddrMan
     {
         LOCK(cs);
         std::vector<int>().swap(vRandom);
-        RAND_bytes(&nKey[0], 32);
+        GetRandBytes(nKey.data(), 32);
         vvTried = std::vector<std::vector<int>>(ADDRMAN_TRIED_BUCKET_COUNT, std::vector<int>(0));
         vvNew = std::vector<std::set<int>>(ADDRMAN_NEW_BUCKET_COUNT, std::set<int>());
         // Will need for Bitcoin rebase

src/crypto/sha512.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016 The Bitcoin Core developers
+// Copyright (c) 2014-2019 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -23,6 +23,7 @@ class CSHA512
     CSHA512& Write(const unsigned char* data, size_t len);
     void Finalize(unsigned char hash[OUTPUT_SIZE]);
     CSHA512& Reset();
+    uint64_t Size() const { return bytes; }
 };
 
 #endif // BITCOIN_CRYPTO_SHA512_H

src/gridcoin/scraper/scraper.cpp

@@ -4,6 +4,7 @@
 
 #include "main.h"
 #include "node/ui_interface.h"
+#include "random.h"
 
 #include "gridcoin/appcache.h"
 #include "gridcoin/beacon.h"

src/gridcoin/staking/kernel.cpp

@@ -8,6 +8,7 @@
 #include "gridcoin/staking/kernel.h"
 #include "txdb.h"
 #include "main.h"
+#include "random.h"
 #include "streams.h"
 #include "util.h"
 

src/gridcoin/staking/spam.h

@@ -7,6 +7,7 @@
 
 #include "kernel.h"
 #include "uint256.h"
+#include "random.h"
 
 #include <array>
 #include <cmath>

src/init.cpp

@@ -11,6 +11,7 @@
 #include "txdb.h"
 #include "wallet/walletdb.h"
 #include "banman.h"
+#include "random.h"
 #include "rpc/server.h"
 #include "init.h"
 #include "node/ui_interface.h"
@@ -606,6 +607,10 @@ bool InitSanityCheck(void)
         return false;
     }
 
+    if (!Random_SanityCheck()) {
+        return InitError("OS cryptographic RNG sanity check failure. Aborting.");
+    }
+
     return true;
 }
 
@@ -929,6 +934,7 @@ bool AppInit2(ThreadHandlerPtr threads)
     // Initialize internal hashing code with SSE/AVX2 optimizations. In the future we will also have ARM/NEON optimizations.
     std::string sha256_algo = SHA256AutoDetect();
     LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo);
+    RandomInit();
 
     LogPrintf("Block version 11 hard fork configured for block %d", Params().GetConsensus().BlockV11Height);
 
@@ -1268,12 +1274,10 @@ bool AppInit2(ThreadHandlerPtr threads)
 
     if (fFirstRun)
     {
-        // Create new keyUser and set as default key
-        RandAddSeedPerfmon();
-
         // So Clang doesn't complain, even though we are really essentially single-threaded here.
         LOCK(pwalletMain->cs_wallet);
 
+        // Create new keyUser and set as default key
         CPubKey newDefaultKey;
         if (pwalletMain->GetKeyFromPool(newDefaultKey, false)) {
             pwalletMain->SetDefaultKey(newDefaultKey);
@@ -1385,8 +1389,6 @@ bool AppInit2(ThreadHandlerPtr threads)
     if (!CheckDiskSpace())
         return false;
 
-    RandAddSeedPerfmon();
-
     if (!GRC::Initialize(threads, pindexBest)) {
         return false;
     }
@@ -1426,6 +1428,11 @@ bool AppInit2(ThreadHandlerPtr threads)
     CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler);
     threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop));
 
+    // Gather some entropy once per minute.
+    scheduler.scheduleEvery([]{
+        RandAddPeriodic();
+    }, std::chrono::minutes{1});
+
     // TODO: Do we need this? It would require porting the Bitcoin signal handler.
     // GetMainSignals().RegisterBackgroundSignalScheduler(scheduler);
 

src/main.cpp

@@ -34,6 +34,7 @@
 #include "node/blockstorage.h"
 #include "policy/fees.h"
 #include "policy/policy.h"
+#include "random.h"
 #include "validation.h"
 
 #include <boost/algorithm/string/replace.hpp>
@@ -2918,8 +2919,6 @@ bool static AlreadyHave(CTxDB& txdb, const CInv& inv)
 
 bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived)
 {
-    RandAddSeedPerfmon();
-
     LogPrint(BCLog::LogFlags::NOISY, "received: %s from %s (%" PRIszu " bytes)", strCommand, pfrom->addrName, vRecv.size());
 
     if (strCommand == "aries")
@@ -3782,6 +3781,10 @@ bool ProcessMessages(CNode* pfrom)
         // Checksum
         CDataStream& vRecv = msg.vRecv;
         uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize);
+
+        // We just received a message off the wire, harvest entropy from the time (and the message checksum)
+        RandAddEvent(ReadLE32(hash.begin()));
+
         // TODO: hardcoded checksum size;
         //  will no longer be used once we adopt CNetMessage from Bitcoin
         uint8_t nChecksum[CMessageHeader::CHECKSUM_SIZE];
@@ -3878,7 +3881,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle)
     {
         uint64_t nonce = 0;
         while (nonce == 0) {
-            RAND_bytes((unsigned char*)&nonce, sizeof(nonce));
+            GetRandBytes((unsigned char*)&nonce, sizeof(nonce));
         }
         pto->fPingQueued = false;
         pto->nPingUsecStart = GetTimeMicros();

src/miner.cpp

@@ -21,14 +21,14 @@
 #include "gridcoin/tally.h"
 #include "policy/policy.h"
 #include "policy/fees.h"
+#include "random.h"
 #include "util.h"
 #include "validation.h"
 #include "wallet/wallet.h"
 
 #include <memory>
 #include <algorithm>
 #include <tuple>
-#include <random>
 
 using namespace std;
 
@@ -685,9 +685,7 @@ void SplitCoinStakeOutput(CBlock &blocknew, int64_t &nReward, bool &fEnableStake
     // the percentages is done.
     if (vSideStakeAlloc.size() > nMaxSideStakeOutputs)
     {
-        unsigned int seed = static_cast<unsigned int>(GetAdjustedTime());
-
-        std::shuffle(vSideStakeAlloc.begin(), vSideStakeAlloc.end(), std::default_random_engine(seed));
+        Shuffle(vSideStakeAlloc.begin(), vSideStakeAlloc.end(), FastRandomContext());
     }
 
     // Initialize remaining stake output value to the total value of output for stake, which also includes

src/net.cpp

@@ -12,6 +12,7 @@
 #include "net.h"
 #include "init.h"
 #include "node/ui_interface.h"
+#include "random.h"
 #include "util.h"
 #include "util/threadnames.h"
 
@@ -405,6 +406,10 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest)
         }
 
         pnode->nTimeConnected = GetAdjustedTime();
+
+        // We're making a new connection, harvest entropy from the time (and our peer count)
+        RandAddEvent((uint32_t)pnode->GetId());
+
         return pnode;
     }
     else
@@ -481,7 +486,7 @@ void CNode::PushVersion()
     int64_t nTime = GetAdjustedTime();
     CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService("0.0.0.0",0)));
     CAddress addrMe = GetLocalAddress(&addr);
-    RAND_bytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce));
+    GetRandBytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce));
     LogPrint(BCLog::LogFlags::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%s",
         PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), addrYou.ToString(), addr.ToString());
 
@@ -966,6 +971,9 @@ void ThreadSocketHandler2(void* parg)
                     LOCK(cs_vNodes);
                     vNodes.push_back(pnode);
                 }
+
+                // We received a new connection, harvest entropy from the time (and our peer count)
+                RandAddEvent((uint32_t)pnode->GetId());
             }
         }
 

src/net.h

@@ -9,7 +9,6 @@
 #include <array>
 #include <boost/thread.hpp>
 #include <atomic>
-#include <openssl/rand.h>
 
 #include "netbase.h"
 #include "mruset.h"

src/qt/winshutdownmonitor.cpp

@@ -12,8 +12,6 @@
 
 #include <QDebug>
 
-#include <openssl/rand.h>
-
 // If we don't want a message to be processed by Qt, return true and set result to
 // the value that the window procedure should return. Otherwise return false.
 bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult)