From 755fb9c9a41a4bb1f05128489adbacdfd9be5b3f Mon Sep 17 00:00:00 2001 From: div72 <60045611+div72@users.noreply.github.com> Date: Wed, 20 Oct 2021 23:25:37 +0300 Subject: [PATCH 1/4] crypto: add CSHA512::Size() --- src/crypto/sha512.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index c5c085831a..10af291d0c 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ class CSHA512 CSHA512& Write(const unsigned char* data, size_t len); void Finalize(unsigned char hash[OUTPUT_SIZE]); CSHA512& Reset(); + uint64_t Size() const { return bytes; } }; #endif // BITCOIN_CRYPTO_SHA512_H From 4372e712a018af071c779b06f6bb656e9e663183 Mon Sep 17 00:00:00 2001 From: div72 <60045611+div72@users.noreply.github.com> Date: Wed, 20 Oct 2021 23:01:09 +0300 Subject: [PATCH 2/4] random: port upstream random changes --- src/Makefile.am | 4 + src/Makefile.test.include | 1 + src/random.cpp | 716 ++++++++++++++++++++++++++++++++++++++ src/random.h | 268 ++++++++++++++ src/randomenv.cpp | 500 ++++++++++++++++++++++++++ src/randomenv.h | 17 + src/test/random_tests.cpp | 138 ++++++++ src/util.cpp | 152 -------- 8 files changed, 1644 insertions(+), 152 deletions(-) create mode 100644 src/random.cpp create mode 100644 src/random.h create mode 100644 src/randomenv.cpp create mode 100644 src/randomenv.h create mode 100644 src/test/random_tests.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 11229f079d..7df5baa5d7 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -156,6 +156,8 @@ GRIDCOIN_CORE_H = \ prevector.h \ primitives/transaction.h \ protocol.h \ + random.h \ + randomenv.h \ reverselock.h \ rpc/blockchain.h \ rpc/client.h \ @@ -257,6 +259,8 @@ GRIDCOIN_CORE_CPP = addrdb.cpp \ policy/policy.cpp \ primitives/transaction.cpp \ protocol.cpp \ + random.cpp \ + randomenv.cpp \ rpc/blockchain.cpp \ rpc/client.cpp \ rpc/dataacq.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index efe95b8825..e6d3a31a0b 100755 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -56,6 +56,7 @@ GRIDCOIN_TESTS =\ test/mruset_tests.cpp \ test/multisig_tests.cpp \ test/netbase_tests.cpp \ + test/random_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ diff --git a/src/random.cpp b/src/random.cpp new file mode 100644 index 0000000000..174f4cef31 --- /dev/null +++ b/src/random.cpp @@ -0,0 +1,716 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#ifdef WIN32 +#include // for Windows API +#include +#endif +#include // for LogPrintf() +#include +#include +#include // for Mutex +#include // for GetTimeMicros() + +#include +#include + +#ifndef WIN32 +#include +#include +#endif + +#ifdef HAVE_SYS_GETRANDOM +#include +#include +#endif +#if defined(HAVE_GETENTROPY) || (defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX)) +#include +#endif +#if defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) +#include +#endif +#ifdef HAVE_SYSCTL_ARND +#include +#endif + +[[noreturn]] static void RandFailure() +{ + LogPrintf("Failed to read randomness, aborting\n"); + std::abort(); +} + +static inline int64_t GetPerformanceCounter() noexcept +{ + // Read the hardware time stamp counter when available. + // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + return __rdtsc(); +#elif !defined(_MSC_VER) && defined(__i386__) + uint64_t r = 0; + __asm__ volatile ("rdtsc" : "=A"(r)); // Constrain the r variable to the eax:edx pair. + return r; +#elif !defined(_MSC_VER) && (defined(__x86_64__) || defined(__amd64__)) + uint64_t r1 = 0, r2 = 0; + __asm__ volatile ("rdtsc" : "=a"(r1), "=d"(r2)); // Constrain r1 to rax and r2 to rdx. + return (r2 << 32) | r1; +#else + // Fall back to using C++11 clock (usually microsecond or nanosecond precision) + return std::chrono::high_resolution_clock::now().time_since_epoch().count(); +#endif +} + +#ifdef HAVE_GETCPUID +static bool g_rdrand_supported = false; +static bool g_rdseed_supported = false; +static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; +static constexpr uint32_t CPUID_F7_EBX_RDSEED = 0x00040000; +#ifdef bit_RDRND +static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND"); +#endif +#ifdef bit_RDSEED +static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); +#endif + +static void InitHardwareRand() +{ + uint32_t eax, ebx, ecx, edx; + GetCPUID(1, 0, eax, ebx, ecx, edx); + if (ecx & CPUID_F1_ECX_RDRAND) { + g_rdrand_supported = true; + } + GetCPUID(7, 0, eax, ebx, ecx, edx); + if (ebx & CPUID_F7_EBX_RDSEED) { + g_rdseed_supported = true; + } +} + +static void ReportHardwareRand() +{ + // This must be done in a separate function, as InitHardwareRand() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rdseed_supported) { + LogPrintf("Using RdSeed as additional entropy source\n"); + } + if (g_rdrand_supported) { + LogPrintf("Using RdRand as an additional entropy source\n"); + } +} + +/** Read 64 bits of entropy using rdrand. + * + * Must only be called when RdRand is supported. + */ +static uint64_t GetRdRand() noexcept +{ + // RdRand may very rarely fail. Invoke it up to 10 times in a loop to reduce this risk. +#ifdef __i386__ + uint8_t ok; + // Initialize to 0 to silence a compiler warning that r1 or r2 may be used + // uninitialized. Even if rdrand fails (!ok) it will set the output to 0, + // but there is no way that the compiler could know that. + uint32_t r1 = 0, r2 = 0; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1 = 0; // See above why we initialize to 0. + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %rax + if (ok) break; + } + return r1; +#else +#error "RdRand is only supported on x86 and x86_64" +#endif +} + +/** Read 64 bits of entropy using rdseed. + * + * Must only be called when RdSeed is supported. + */ +static uint64_t GetRdSeed() noexcept +{ + // RdSeed may fail when the HW RNG is overloaded. Loop indefinitely until enough entropy is gathered, + // but pause after every failure. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + do { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %rax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return r1; +#else +#error "RdSeed is only supported on x86 and x86_64" +#endif +} + +#else +/* Access to other hardware random number generators could be added here later, + * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). + * Slower sources should probably be invoked separately, and/or only from + * RandAddPeriodic (which is called once a minute). + */ +static void InitHardwareRand() {} +static void ReportHardwareRand() {} +#endif + +/** Add 64 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareFast(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + if (g_rdrand_supported) { + uint64_t out = GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } +#endif +} + +/** Add 256 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareSlow(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + // When we want 256 bits of entropy, prefer RdSeed over RdRand, as it's + // guaranteed to produce independent randomness on every call. + if (g_rdseed_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRdSeed(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } + // When falling back to RdRand, XOR the result of 1024 results. + // This guarantees a reseeding occurs between each. + if (g_rdrand_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = 0; + for (int j = 0; j < 1024; ++j) out ^= GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } +#endif +} + +/** Use repeated SHA512 to strengthen the randomness in seed32, and feed into hasher. */ +static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA512& hasher) noexcept +{ + CSHA512 inner_hasher; + inner_hasher.Write(seed, sizeof(seed)); + + // Hash loop + unsigned char buffer[64]; + int64_t stop = GetTimeMicros() + microseconds; + do { + for (int i = 0; i < 1000; ++i) { + inner_hasher.Finalize(buffer); + inner_hasher.Reset(); + inner_hasher.Write(buffer, sizeof(buffer)); + } + // Benchmark operation and feed it into outer hasher. + int64_t perf = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perf, sizeof(perf)); + } while (GetTimeMicros() < stop); + + // Produce output from inner state and feed it to outer hasher. + inner_hasher.Finalize(buffer); + hasher.Write(buffer, sizeof(buffer)); + // Try to clean up. + inner_hasher.Reset(); + memory_cleanse(buffer, sizeof(buffer)); +} + +#ifndef WIN32 +/** Fallback: get 32 bytes of system entropy from /dev/urandom. The most + * compatible way to get cryptographic randomness on UNIX-ish platforms. + */ +static void GetDevURandom(unsigned char *ent32) +{ + int f = open("/dev/urandom", O_RDONLY); + if (f == -1) { + RandFailure(); + } + int have = 0; + do { + ssize_t n = read(f, ent32 + have, NUM_OS_RANDOM_BYTES - have); + if (n <= 0 || n + have > NUM_OS_RANDOM_BYTES) { + close(f); + RandFailure(); + } + have += n; + } while (have < NUM_OS_RANDOM_BYTES); + close(f); +} +#endif + +/** Get 32 bytes of system entropy. */ +void GetOSRand(unsigned char *ent32) +{ +#if defined(WIN32) + HCRYPTPROV hProvider; + int ret = CryptAcquireContextW(&hProvider, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT); + if (!ret) { + RandFailure(); + } + ret = CryptGenRandom(hProvider, NUM_OS_RANDOM_BYTES, ent32); + if (!ret) { + RandFailure(); + } + CryptReleaseContext(hProvider, 0); +#elif defined(HAVE_SYS_GETRANDOM) + /* Linux. From the getrandom(2) man page: + * "If the urandom source has been initialized, reads of up to 256 bytes + * will always return as many bytes as requested and will not be + * interrupted by signals." + */ + int rv = syscall(SYS_getrandom, ent32, NUM_OS_RANDOM_BYTES, 0); + if (rv != NUM_OS_RANDOM_BYTES) { + if (rv < 0 && errno == ENOSYS) { + /* Fallback for kernel <3.17: the return value will be -1 and errno + * ENOSYS if the syscall is not available, in that case fall back + * to /dev/urandom. + */ + GetDevURandom(ent32); + } else { + RandFailure(); + } + } +#elif defined(HAVE_GETENTROPY) && defined(__OpenBSD__) + /* On OpenBSD this can return up to 256 bytes of entropy, will return an + * error if more are requested. + * The call cannot return less than the requested number of bytes. + getentropy is explicitly limited to openbsd here, as a similar (but not + the same) function may exist on other platforms via glibc. + */ + if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { + RandFailure(); + } + // Silence a compiler warning about unused function. + (void)GetDevURandom; +#elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) + /* getentropy() is available on macOS 10.12 and later. + */ + if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { + RandFailure(); + } + // Silence a compiler warning about unused function. + (void)GetDevURandom; +#elif defined(HAVE_SYSCTL_ARND) + /* FreeBSD, NetBSD and similar. It is possible for the call to return less + * bytes than requested, so need to read in a loop. + */ + static int name[2] = {CTL_KERN, KERN_ARND}; + int have = 0; + do { + size_t len = NUM_OS_RANDOM_BYTES - have; + if (sysctl(name, std::size(name), ent32 + have, &len, nullptr, 0) != 0) { + RandFailure(); + } + have += len; + } while (have < NUM_OS_RANDOM_BYTES); + // Silence a compiler warning about unused function. + (void)GetDevURandom; +#else + /* Fall back to /dev/urandom if there is no specific method implemented to + * get system entropy for this OS. + */ + GetDevURandom(ent32); +#endif +} + +namespace { + +class RNGState { + Mutex m_mutex; + /* The RNG state consists of 256 bits of entropy, taken from the output of + * one operation's SHA512 output, and fed as input to the next one. + * Carrying 256 bits of entropy should be sufficient to guarantee + * unpredictability as long as any entropy source was ever unpredictable + * to an attacker. To protect against situations where an attacker might + * observe the RNG's state, fresh entropy is always mixed when + * GetStrongRandBytes is called. + */ + unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; + uint64_t m_counter GUARDED_BY(m_mutex) = 0; + bool m_strongly_seeded GUARDED_BY(m_mutex) = false; + + Mutex m_events_mutex; + CSHA256 m_events_hasher GUARDED_BY(m_events_mutex); + +public: + RNGState() noexcept + { + InitHardwareRand(); + } + + ~RNGState() + { + } + + void AddEvent(uint32_t event_info) noexcept + { + LOCK(m_events_mutex); + + m_events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); + // Get the low four bytes of the performance counter. This translates to roughly the + // subsecond part. + uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); + m_events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); + } + + /** + * Feed (the hash of) all events added through AddEvent() to hasher. + */ + void SeedEvents(CSHA512& hasher) noexcept + { + // We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, + // since we want it to be fast as network peers may be able to trigger it repeatedly. + LOCK(m_events_mutex); + + unsigned char events_hash[32]; + m_events_hasher.Finalize(events_hash); + hasher.Write(events_hash, 32); + + // Re-initialize the hasher with the finalized state to use later. + m_events_hasher.Reset(); + m_events_hasher.Write(events_hash, 32); + } + + /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. + * + * If this function has never been called with strong_seed = true, false is returned. + */ + bool MixExtract(unsigned char* out, size_t num, CSHA512&& hasher, bool strong_seed) noexcept + { + assert(num <= 32); + unsigned char buf[64]; + static_assert(sizeof(buf) == CSHA512::OUTPUT_SIZE, "Buffer needs to have hasher's output size"); + bool ret; + { + LOCK(m_mutex); + ret = (m_strongly_seeded |= strong_seed); + // Write the current state of the RNG into the hasher + hasher.Write(m_state, 32); + // Write a new counter number into the state + hasher.Write((const unsigned char*)&m_counter, sizeof(m_counter)); + ++m_counter; + // Finalize the hasher + hasher.Finalize(buf); + // Store the last 32 bytes of the hash output as new RNG state. + memcpy(m_state, buf + 32, 32); + } + // If desired, copy (up to) the first 32 bytes of the hash output as output. + if (num) { + assert(out != nullptr); + memcpy(out, buf, num); + } + // Best effort cleanup of internal state + hasher.Reset(); + memory_cleanse(buf, 64); + return ret; + } +}; + +RNGState& GetRNGState() noexcept +{ + // This C++11 idiom relies on the guarantee that static variable are initialized + // on first call, even when multiple parallel calls are permitted. + static std::vector> g_rng(1); + return g_rng[0]; +} +} + +/* A note on the use of noexcept in the seeding functions below: + * + * None of the RNG code should ever throw any exception. + */ + +static void SeedTimestamp(CSHA512& hasher) noexcept +{ + int64_t perfcounter = GetPerformanceCounter(); + hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); +} + +static void SeedFast(CSHA512& hasher) noexcept +{ + unsigned char buffer[32]; + + // Stack pointer to indirectly commit to thread/callstack + const unsigned char* ptr = buffer; + hasher.Write((const unsigned char*)&ptr, sizeof(ptr)); + + // Hardware randomness is very fast when available; use it always. + SeedHardwareFast(hasher); + + // High-precision timestamp + SeedTimestamp(hasher); +} + +static void SeedSlow(CSHA512& hasher, RNGState& rng) noexcept +{ + unsigned char buffer[32]; + + // Everything that the 'fast' seeder includes + SeedFast(hasher); + + // OS randomness + GetOSRand(buffer); + hasher.Write(buffer, sizeof(buffer)); + + // Add the events hasher into the mix + rng.SeedEvents(hasher); + + // High-precision timestamp. + // + // Note that we also commit to a timestamp in the Fast seeder, so we indirectly commit to a + // benchmark of all the entropy gathering sources in this function). + SeedTimestamp(hasher); +} + +/** Extract entropy from rng, strengthen it, and feed it into hasher. */ +static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noexcept +{ + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen the seed, and feed it into hasher. + Strengthen(strengthen_seed, microseconds, hasher); +} + +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept +{ + // Everything that the 'fast' seeder includes + SeedFast(hasher); + + // High-precision timestamp + SeedTimestamp(hasher); + + // Add the events hasher into the mix + rng.SeedEvents(hasher); + + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); + LogPrint(BCLog::RAND, "Feeding %i bytes of dynamic environment data into RNG\n", hasher.Size() - old_size); + + // Strengthen for 10 ms + SeedStrengthen(hasher, rng, 10000); +} + +static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept +{ + // Gather 256 bits of hardware randomness, if available + SeedHardwareSlow(hasher); + + // Everything that the 'slow' seeder includes. + SeedSlow(hasher, rng); + + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); + + // Static environment data + RandAddStaticEnv(hasher); + LogPrint(BCLog::RAND, "Feeding %i bytes of environment data into RNG\n", hasher.Size() - old_size); + + // Strengthen for 100 ms + SeedStrengthen(hasher, rng, 100000); +} + +enum class RNGLevel { + FAST, //!< Automatically called by GetRandBytes + SLOW, //!< Automatically called by GetStrongRandBytes + PERIODIC, //!< Called by RandAddPeriodic() +}; + +static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept +{ + // Make sure the RNG is initialized first (as all Seed* function possibly need hwrand to be available). + RNGState& rng = GetRNGState(); + + assert(num <= 32); + + CSHA512 hasher; + switch (level) { + case RNGLevel::FAST: + SeedFast(hasher); + break; + case RNGLevel::SLOW: + SeedSlow(hasher, rng); + break; + case RNGLevel::PERIODIC: + SeedPeriodic(hasher, rng); + break; + } + + // Combine with and update state + if (!rng.MixExtract(out, num, std::move(hasher), false)) { + // On the first invocation, also seed with SeedStartup(). + CSHA512 startup_hasher; + SeedStartup(startup_hasher, rng); + rng.MixExtract(out, num, std::move(startup_hasher), true); + } +} + +void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } +void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } +void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } + +bool g_mock_deterministic_tests{false}; + +uint64_t GetRand(uint64_t nMax) noexcept +{ + return FastRandomContext(g_mock_deterministic_tests).randrange(nMax); +} + +int GetRandInt(int nMax) noexcept +{ + return GetRand(nMax); +} + +uint256 GetRandHash() noexcept +{ + uint256 hash; + GetRandBytes((unsigned char*)&hash, sizeof(hash)); + return hash; +} + +void FastRandomContext::RandomSeed() +{ + uint256 seed = GetRandHash(); + rng.SetKey(seed.begin(), 32); + requires_seed = false; +} + +uint256 FastRandomContext::rand256() noexcept +{ + if (bytebuf_size < 32) { + FillByteBuffer(); + } + uint256 ret; + memcpy(ret.begin(), bytebuf + 64 - bytebuf_size, 32); + bytebuf_size -= 32; + return ret; +} + +std::vector FastRandomContext::randbytes(size_t len) +{ + if (requires_seed) RandomSeed(); + std::vector ret(len); + if (len > 0) { + rng.Keystream(ret.data(), len); + } + return ret; +} + +FastRandomContext::FastRandomContext(const uint256& seed) noexcept : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +{ + rng.SetKey(seed.begin(), 32); +} + +bool Random_SanityCheck() +{ + uint64_t start = GetPerformanceCounter(); + + /* This does not measure the quality of randomness, but it does test that + * GetOSRand() overwrites all 32 bytes of the output given a maximum + * number of tries. + */ + static const ssize_t MAX_TRIES = 1024; + uint8_t data[NUM_OS_RANDOM_BYTES]; + bool overwritten[NUM_OS_RANDOM_BYTES] = {}; /* Tracks which bytes have been overwritten at least once */ + int num_overwritten; + int tries = 0; + /* Loop until all bytes have been overwritten at least once, or max number tries reached */ + do { + memset(data, 0, NUM_OS_RANDOM_BYTES); + GetOSRand(data); + for (int x=0; x < NUM_OS_RANDOM_BYTES; ++x) { + overwritten[x] |= (data[x] != 0); + } + + num_overwritten = 0; + for (int x=0; x < NUM_OS_RANDOM_BYTES; ++x) { + if (overwritten[x]) { + num_overwritten += 1; + } + } + + tries += 1; + } while (num_overwritten < NUM_OS_RANDOM_BYTES && tries < MAX_TRIES); + if (num_overwritten != NUM_OS_RANDOM_BYTES) return false; /* If this failed, bailed out after too many tries */ + + // Check that GetPerformanceCounter increases at least during a GetOSRand() call + 1ms sleep. + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + uint64_t stop = GetPerformanceCounter(); + if (stop == start) return false; + + // We called GetPerformanceCounter. Use it as entropy. + CSHA512 to_add; + to_add.Write((const unsigned char*)&start, sizeof(start)); + to_add.Write((const unsigned char*)&stop, sizeof(stop)); + GetRNGState().MixExtract(nullptr, 0, std::move(to_add), false); + + return true; +} + +FastRandomContext::FastRandomContext(bool fDeterministic) noexcept : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +{ + if (!fDeterministic) { + return; + } + uint256 seed; + rng.SetKey(seed.begin(), 32); +} + +FastRandomContext& FastRandomContext::operator=(FastRandomContext&& from) noexcept +{ + requires_seed = from.requires_seed; + rng = from.rng; + std::copy(std::begin(from.bytebuf), std::end(from.bytebuf), std::begin(bytebuf)); + bytebuf_size = from.bytebuf_size; + bitbuf = from.bitbuf; + bitbuf_size = from.bitbuf_size; + from.requires_seed = true; + from.bytebuf_size = 0; + from.bitbuf_size = 0; + return *this; +} + +void RandomInit() +{ + // Invoke RNG code to trigger initialization (if not already performed) + ProcRand(nullptr, 0, RNGLevel::FAST); + + ReportHardwareRand(); +} diff --git a/src/random.h b/src/random.h new file mode 100644 index 0000000000..0c6dc24983 --- /dev/null +++ b/src/random.h @@ -0,0 +1,268 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOM_H +#define BITCOIN_RANDOM_H + +#include +#include +#include + +#include // For std::chrono::microseconds +#include +#include + +/** + * Overall design of the RNG and entropy sources. + * + * We maintain a single global 256-bit RNG state for all high-quality randomness. + * The following (classes of) functions interact with that state by mixing in new + * entropy, and optionally extracting random output from it: + * + * - The GetRand*() class of functions, as well as construction of FastRandomContext objects, + * perform 'fast' seeding, consisting of mixing in: + * - A stack pointer (indirectly committing to calling thread and call stack) + * - A high-precision timestamp (rdtsc when available, c++ high_resolution_clock otherwise) + * - 64 bits from the hardware RNG (rdrand) when available. + * These entropy sources are very fast, and only designed to protect against situations + * where a VM state restore/copy results in multiple systems with the same randomness. + * FastRandomContext on the other hand does not protect against this once created, but + * is even faster (and acceptable to use inside tight loops). + * + * - The GetStrongRand*() class of function perform 'slow' seeding, including everything + * that fast seeding includes, but additionally: + * - OS entropy (/dev/urandom, getrandom(), ...). The application will terminate if + * this entropy source fails. + * - Another high-precision timestamp (indirectly committing to a benchmark of all the + * previous sources). + * These entropy sources are slower, but designed to make sure the RNG state contains + * fresh data that is unpredictable to attackers. + * + * - RandAddPeriodic() seeds everything that fast seeding includes, but additionally: + * - A high-precision timestamp + * - Dynamic environment data (performance monitoring, ...) + * - Strengthen the entropy for 10 ms using repeated SHA512. + * This is run once every minute. + * + * On first use of the RNG (regardless of what function is called first), all entropy + * sources used in the 'slow' seeder are included, but also: + * - 256 bits from the hardware RNG (rdseed or rdrand) when available. + * - Dynamic environment data (performance monitoring, ...) + * - Static environment data + * - Strengthen the entropy for 100 ms using repeated SHA512. + * + * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and + * (up to) the first 32 bytes of H are produced as output, while the last 32 bytes + * become the new RNG state. +*/ + +/** + * Generate random data via the internal PRNG. + * + * These functions are designed to be fast (sub microsecond), but do not necessarily + * meaningfully add entropy to the PRNG state. + * + * Thread-safe. + */ +void GetRandBytes(unsigned char* buf, int num) noexcept; +/** Generate a uniform random integer in the range [0..range). Precondition: range > 0 */ +uint64_t GetRand(uint64_t nMax) noexcept; +/** Generate a uniform random duration in the range [0..max). Precondition: max.count() > 0 */ +template +D GetRandomDuration(typename std::common_type::type max) noexcept +// Having the compiler infer the template argument from the function argument +// is dangerous, because the desired return value generally has a different +// type than the function argument. So std::common_type is used to force the +// call site to specify the type of the return value. +{ + assert(max.count() > 0); + return D{GetRand(max.count())}; +}; +constexpr auto GetRandMicros = GetRandomDuration; +constexpr auto GetRandMillis = GetRandomDuration; +int GetRandInt(int nMax) noexcept; +uint256 GetRandHash() noexcept; + +/** + * Gather entropy from various sources, feed it into the internal PRNG, and + * generate random data using it. + * + * This function will cause failure whenever the OS RNG fails. + * + * Thread-safe. + */ +void GetStrongRandBytes(unsigned char* buf, int num) noexcept; + +/** + * Gather entropy from various expensive sources, and feed them to the PRNG state. + * + * Thread-safe. + */ +void RandAddPeriodic() noexcept; + +/** + * Gathers entropy from the low bits of the time at which events occur. Should + * be called with a uint32_t describing the event at the time an event occurs. + * + * Thread-safe. + */ +void RandAddEvent(const uint32_t event_info) noexcept; + +/** + * Fast randomness source. This is seeded once with secure random data, but + * is completely deterministic and does not gather more entropy after that. + * + * This class is not thread-safe. + */ +class FastRandomContext +{ +private: + bool requires_seed; + ChaCha20 rng; + + unsigned char bytebuf[64]; + int bytebuf_size; + + uint64_t bitbuf; + int bitbuf_size; + + void RandomSeed(); + + void FillByteBuffer() + { + if (requires_seed) { + RandomSeed(); + } + rng.Keystream(bytebuf, sizeof(bytebuf)); + bytebuf_size = sizeof(bytebuf); + } + + void FillBitBuffer() + { + bitbuf = rand64(); + bitbuf_size = 64; + } + +public: + explicit FastRandomContext(bool fDeterministic = false) noexcept; + + /** Initialize with explicit seed (only for testing) */ + explicit FastRandomContext(const uint256& seed) noexcept; + + // Do not permit copying a FastRandomContext (move it, or create a new one to get reseeded). + FastRandomContext(const FastRandomContext&) = delete; + FastRandomContext(FastRandomContext&&) = delete; + FastRandomContext& operator=(const FastRandomContext&) = delete; + + /** Move a FastRandomContext. If the original one is used again, it will be reseeded. */ + FastRandomContext& operator=(FastRandomContext&& from) noexcept; + + /** Generate a random 64-bit integer. */ + uint64_t rand64() noexcept + { + if (bytebuf_size < 8) FillByteBuffer(); + uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); + bytebuf_size -= 8; + return ret; + } + + /** Generate a random (bits)-bit integer. */ + uint64_t randbits(int bits) noexcept + { + if (bits == 0) { + return 0; + } else if (bits > 32) { + return rand64() >> (64 - bits); + } else { + if (bitbuf_size < bits) FillBitBuffer(); + uint64_t ret = bitbuf & (~(uint64_t)0 >> (64 - bits)); + bitbuf >>= bits; + bitbuf_size -= bits; + return ret; + } + } + + /** Generate a random integer in the range [0..range). + * Precondition: range > 0. + */ + uint64_t randrange(uint64_t range) noexcept + { + assert(range); + --range; + int bits = CountBits(range); + while (true) { + uint64_t ret = randbits(bits); + if (ret <= range) return ret; + } + } + + /** Generate random bytes. */ + std::vector randbytes(size_t len); + + /** Generate a random 32-bit integer. */ + uint32_t rand32() noexcept { return randbits(32); } + + /** generate a random uint256. */ + uint256 rand256() noexcept; + + /** Generate a random boolean. */ + bool randbool() noexcept { return randbits(1); } + + // Compatibility with the C++11 UniformRandomBitGenerator concept + typedef uint64_t result_type; + static constexpr uint64_t min() { return 0; } + static constexpr uint64_t max() { return std::numeric_limits::max(); } + inline uint64_t operator()() noexcept { return rand64(); } +}; + +/** More efficient than using std::shuffle on a FastRandomContext. + * + * This is more efficient as std::shuffle will consume entropy in groups of + * 64 bits at the time and throw away most. + * + * This also works around a bug in libstdc++ std::shuffle that may cause + * type::operator=(type&&) to be invoked on itself, which the library's + * debug mode detects and panics on. This is a known issue, see + * https://stackoverflow.com/questions/22915325/avoiding-self-assignment-in-stdshuffle + */ +template +void Shuffle(I first, I last, R&& rng) +{ + while (first != last) { + size_t j = rng.randrange(last - first); + if (j) { + using std::swap; + swap(*first, *(first + j)); + } + ++first; + } +} + +/* Number of random bytes returned by GetOSRand. + * When changing this constant make sure to change all call sites, and make + * sure that the underlying OS APIs for all platforms support the number. + * (many cap out at 256 bytes). + */ +static const int NUM_OS_RANDOM_BYTES = 32; + +/** Get 32 bytes of system entropy. Do not use this in application code: use + * GetStrongRandBytes instead. + */ +void GetOSRand(unsigned char* ent32); + +/** Check that OS randomness is available and returning the requested number + * of bytes. + */ +bool Random_SanityCheck(); + +/** + * Initialize global RNG state and log any CPU features that are used. + * + * Calling this function is optional. RNG state will be initialized when first + * needed if it is not called. + */ +void RandomInit(); + +#endif // BITCOIN_RANDOM_H diff --git a/src/randomenv.cpp b/src/randomenv.cpp new file mode 100644 index 0000000000..43e0456af4 --- /dev/null +++ b/src/randomenv.cpp @@ -0,0 +1,500 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include + +#include +#include +#include +#include +#include // for GetTime() +#ifdef WIN32 +#include // for Windows API +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#ifndef WIN32 +#include // must go before a number of other headers +#include +#include +#include +#include +#include +#include +#include +#include +#endif +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS +#include +#endif +#if HAVE_SYSCTL +#include +#if HAVE_VM_VM_PARAM_H +#include +#endif +#if HAVE_SYS_RESOURCES_H +#include +#endif +#if HAVE_SYS_VMMETER_H +#include +#endif +#endif +#if defined(HAVE_STRONG_GETAUXVAL) +#include +#endif + +//! Necessary on some platforms +extern char** environ; + +namespace { + +void RandAddSeedPerfmon(CSHA512& hasher) +{ +#ifdef WIN32 + // Seed with the entire set of perfmon data + + // This can take up to 2 seconds, so only do it every 10 minutes. + // Initialize last_perfmon to 0 seconds, we don't skip the first call. + static std::atomic last_perfmon{0s}; + auto last_time = last_perfmon.load(); + auto current_time = GetTime(); + if (current_time < last_time + std::chrono::minutes{10}) return; + last_perfmon = current_time; + + std::vector vData(250000, 0); + long ret = 0; + unsigned long nSize = 0; + const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data + while (true) { + nSize = vData.size(); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); + if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) + break; + vData.resize(std::min((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially + } + RegCloseKey(HKEY_PERFORMANCE_DATA); + if (ret == ERROR_SUCCESS) { + hasher.Write(vData.data(), nSize); + memory_cleanse(vData.data(), nSize); + } else { + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it is isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. + } +#endif +} + +/** Helper to easily feed data into a CSHA512. + * + * Note that this does not serialize the passed object (like stream.h's << operators do). + * Its raw memory representation is used directly. + */ +template +CSHA512& operator<<(CSHA512& hasher, const T& data) { + static_assert(!std::is_same::type, char*>::value, "Calling operator<<(CSHA512, char*) is probably not what you want"); + static_assert(!std::is_same::type, unsigned char*>::value, "Calling operator<<(CSHA512, unsigned char*) is probably not what you want"); + static_assert(!std::is_same::type, const char*>::value, "Calling operator<<(CSHA512, const char*) is probably not what you want"); + static_assert(!std::is_same::type, const unsigned char*>::value, "Calling operator<<(CSHA512, const unsigned char*) is probably not what you want"); + hasher.Write((const unsigned char*)&data, sizeof(data)); + return hasher; +} + +#ifndef WIN32 +void AddSockaddr(CSHA512& hasher, const struct sockaddr *addr) +{ + if (addr == nullptr) return; + switch (addr->sa_family) { + case AF_INET: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in)); + break; + case AF_INET6: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in6)); + break; + default: + hasher.Write((const unsigned char*)&addr->sa_family, sizeof(addr->sa_family)); + } +} + +void AddFile(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + int f = open(path, O_RDONLY); + size_t total = 0; + if (f != -1) { + unsigned char fbuf[4096]; + int n; + hasher.Write((const unsigned char*)&f, sizeof(f)); + if (fstat(f, &sb) == 0) hasher << sb; + do { + n = read(f, fbuf, sizeof(fbuf)); + if (n > 0) hasher.Write(fbuf, n); + total += n; + /* not bothering with EINTR handling. */ + } while (n == sizeof(fbuf) && total < 1048576); // Read only the first 1 Mbyte + close(f); + } +} + +void AddPath(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + if (stat(path, &sb) == 0) { + hasher.Write((const unsigned char*)path, strlen(path) + 1); + hasher << sb; + } +} +#endif + +#if HAVE_SYSCTL +template +void AddSysctl(CSHA512& hasher) +{ + int CTL[sizeof...(S)] = {S...}; + unsigned char buffer[65536]; + size_t siz = 65536; + int ret = sysctl(CTL, sizeof...(S), buffer, &siz, nullptr, 0); + if (ret == 0 || (ret == -1 && errno == ENOMEM)) { + hasher << sizeof(CTL); + hasher.Write((const unsigned char*)CTL, sizeof(CTL)); + if (siz > sizeof(buffer)) siz = sizeof(buffer); + hasher << siz; + hasher.Write(buffer, siz); + } +} +#endif + +#ifdef HAVE_GETCPUID +void inline AddCPUID(CSHA512& hasher, uint32_t leaf, uint32_t subleaf, uint32_t& ax, uint32_t& bx, uint32_t& cx, uint32_t& dx) +{ + GetCPUID(leaf, subleaf, ax, bx, cx, dx); + hasher << leaf << subleaf << ax << bx << cx << dx; +} + +void AddAllCPUID(CSHA512& hasher) +{ + uint32_t ax, bx, cx, dx; + // Iterate over all standard leaves + AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax + uint32_t max = ax; + for (uint32_t leaf = 1; leaf <= max && leaf <= 0xFF; ++leaf) { + uint32_t maxsub = 0; + for (uint32_t subleaf = 0; subleaf <= 0xFF; ++subleaf) { + AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); + // Iterate subleafs for leaf values 4, 7, 11, 13 + if (leaf == 4) { + if ((ax & 0x1f) == 0) break; + } else if (leaf == 7) { + if (subleaf == 0) maxsub = ax; + if (subleaf == maxsub) break; + } else if (leaf == 11) { + if ((cx & 0xff00) == 0) break; + } else if (leaf == 13) { + if (ax == 0 && bx == 0 && cx == 0 && dx == 0) break; + } else { + // For any other leaf, stop after subleaf 0. + break; + } + } + } + // Iterate over all extended leaves + AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax + uint32_t ext_max = ax; + for (uint32_t leaf = 0x80000001; leaf <= ext_max && leaf <= 0x800000FF; ++leaf) { + AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); + } +} +#endif +} // namespace + +void RandAddDynamicEnv(CSHA512& hasher) +{ + RandAddSeedPerfmon(hasher); + + // Various clocks +#ifdef WIN32 + FILETIME ftime; + GetSystemTimeAsFileTime(&ftime); + hasher << ftime; +#else + struct timespec ts = {}; +# ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &ts); + hasher << ts; +# endif +# ifdef CLOCK_REALTIME + clock_gettime(CLOCK_REALTIME, &ts); + hasher << ts; +# endif +# ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &ts); + hasher << ts; +# endif + // gettimeofday is available on all UNIX systems, but only has microsecond precision. + struct timeval tv = {}; + gettimeofday(&tv, nullptr); + hasher << tv; +#endif + // Probably redundant, but also use all the clocks C++11 provides: + hasher << std::chrono::system_clock::now().time_since_epoch().count(); + hasher << std::chrono::steady_clock::now().time_since_epoch().count(); + hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); + +#ifndef WIN32 + // Current resource usage. + struct rusage usage = {}; + if (getrusage(RUSAGE_SELF, &usage) == 0) hasher << usage; +#endif + +#ifdef __linux__ + AddFile(hasher, "/proc/diskstats"); + AddFile(hasher, "/proc/vmstat"); + AddFile(hasher, "/proc/schedstat"); + AddFile(hasher, "/proc/zoneinfo"); + AddFile(hasher, "/proc/meminfo"); + AddFile(hasher, "/proc/softirqs"); + AddFile(hasher, "/proc/stat"); + AddFile(hasher, "/proc/self/schedstat"); + AddFile(hasher, "/proc/self/status"); +#endif + +#if HAVE_SYSCTL +# ifdef CTL_KERN +# if defined(KERN_PROC) && defined(KERN_PROC_ALL) + AddSysctl(hasher); +# endif +# endif +# ifdef CTL_HW +# ifdef HW_DISKSTATS + AddSysctl(hasher); +# endif +# endif +# ifdef CTL_VM +# ifdef VM_LOADAVG + AddSysctl(hasher); +# endif +# ifdef VM_TOTAL + AddSysctl(hasher); +# endif +# ifdef VM_METER + AddSysctl(hasher); +# endif +# endif +#endif + + // Stack and heap location + void* addr = malloc(4097); + hasher << &addr << addr; + free(addr); +} + +void RandAddStaticEnv(CSHA512& hasher) +{ + // Some compile-time static properties + hasher << (CHAR_MIN < 0) << sizeof(void*) << sizeof(long) << sizeof(int); +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + hasher << __GNUC__ << __GNUC_MINOR__ << __GNUC_PATCHLEVEL__; +#endif +#ifdef _MSC_VER + hasher << _MSC_VER; +#endif + hasher << __cplusplus; +#ifdef _XOPEN_VERSION + hasher << _XOPEN_VERSION; +#endif +#ifdef __VERSION__ + const char* COMPILER_VERSION = __VERSION__; + hasher.Write((const unsigned char*)COMPILER_VERSION, strlen(COMPILER_VERSION) + 1); +#endif + + // Bitcoin client version + hasher << CLIENT_VERSION; + +#if defined(HAVE_STRONG_GETAUXVAL) + // Information available through getauxval() +# ifdef AT_HWCAP + hasher << getauxval(AT_HWCAP); +# endif +# ifdef AT_HWCAP2 + hasher << getauxval(AT_HWCAP2); +# endif +# ifdef AT_RANDOM + const unsigned char* random_aux = (const unsigned char*)getauxval(AT_RANDOM); + if (random_aux) hasher.Write(random_aux, 16); +# endif +# ifdef AT_PLATFORM + const char* platform_str = (const char*)getauxval(AT_PLATFORM); + if (platform_str) hasher.Write((const unsigned char*)platform_str, strlen(platform_str) + 1); +# endif +# ifdef AT_EXECFN + const char* exec_str = (const char*)getauxval(AT_EXECFN); + if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); +# endif +#endif // HAVE_STRONG_GETAUXVAL + +#ifdef HAVE_GETCPUID + AddAllCPUID(hasher); +#endif + + // Memory locations + hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; + + // Hostname + char hname[256]; + if (gethostname(hname, 256) == 0) { + hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); + } + +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS + // Network interfaces + struct ifaddrs *ifad = NULL; + getifaddrs(&ifad); + struct ifaddrs *ifit = ifad; + while (ifit != NULL) { + hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); + hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); + hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); + AddSockaddr(hasher, ifit->ifa_addr); + AddSockaddr(hasher, ifit->ifa_netmask); + AddSockaddr(hasher, ifit->ifa_dstaddr); + ifit = ifit->ifa_next; + } + freeifaddrs(ifad); +#endif + +#ifndef WIN32 + // UNIX kernel information + struct utsname name; + if (uname(&name) != -1) { + hasher.Write((const unsigned char*)&name.sysname, strlen(name.sysname) + 1); + hasher.Write((const unsigned char*)&name.nodename, strlen(name.nodename) + 1); + hasher.Write((const unsigned char*)&name.release, strlen(name.release) + 1); + hasher.Write((const unsigned char*)&name.version, strlen(name.version) + 1); + hasher.Write((const unsigned char*)&name.machine, strlen(name.machine) + 1); + } + + /* Path and filesystem provided data */ + AddPath(hasher, "/"); + AddPath(hasher, "."); + AddPath(hasher, "/tmp"); + AddPath(hasher, "/home"); + AddPath(hasher, "/proc"); +#ifdef __linux__ + AddFile(hasher, "/proc/cmdline"); + AddFile(hasher, "/proc/cpuinfo"); + AddFile(hasher, "/proc/version"); +#endif + AddFile(hasher, "/etc/passwd"); + AddFile(hasher, "/etc/group"); + AddFile(hasher, "/etc/hosts"); + AddFile(hasher, "/etc/resolv.conf"); + AddFile(hasher, "/etc/timezone"); + AddFile(hasher, "/etc/localtime"); +#endif + + // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these + // will exist on every system. +#if HAVE_SYSCTL +# ifdef CTL_HW +# ifdef HW_MACHINE + AddSysctl(hasher); +# endif +# ifdef HW_MODEL + AddSysctl(hasher); +# endif +# ifdef HW_NCPU + AddSysctl(hasher); +# endif +# ifdef HW_PHYSMEM + AddSysctl(hasher); +# endif +# ifdef HW_USERMEM + AddSysctl(hasher); +# endif +# ifdef HW_MACHINE_ARCH + AddSysctl(hasher); +# endif +# ifdef HW_REALMEM + AddSysctl(hasher); +# endif +# ifdef HW_CPU_FREQ + AddSysctl(hasher); +# endif +# ifdef HW_BUS_FREQ + AddSysctl(hasher); +# endif +# ifdef HW_CACHELINE + AddSysctl(hasher); +# endif +# endif +# ifdef CTL_KERN +# ifdef KERN_BOOTFILE + AddSysctl(hasher); +# endif +# ifdef KERN_BOOTTIME + AddSysctl(hasher); +# endif +# ifdef KERN_CLOCKRATE + AddSysctl(hasher); +# endif +# ifdef KERN_HOSTID + AddSysctl(hasher); +# endif +# ifdef KERN_HOSTUUID + AddSysctl(hasher); +# endif +# ifdef KERN_HOSTNAME + AddSysctl(hasher); +# endif +# ifdef KERN_OSRELDATE + AddSysctl(hasher); +# endif +# ifdef KERN_OSRELEASE + AddSysctl(hasher); +# endif +# ifdef KERN_OSREV + AddSysctl(hasher); +# endif +# ifdef KERN_OSTYPE + AddSysctl(hasher); +# endif +# ifdef KERN_POSIX1 + AddSysctl(hasher); +# endif +# ifdef KERN_VERSION + AddSysctl(hasher); +# endif +# endif +#endif + + // Env variables + if (environ) { + for (size_t i = 0; environ[i]; ++i) { + hasher.Write((const unsigned char*)environ[i], strlen(environ[i])); + } + } + + // Process, thread, user, session, group, ... ids. +#ifdef WIN32 + hasher << GetCurrentProcessId() << GetCurrentThreadId(); +#else + hasher << getpid() << getppid() << getsid(0) << getpgid(0) << getuid() << geteuid() << getgid() << getegid(); +#endif + hasher << std::this_thread::get_id(); +} diff --git a/src/randomenv.h b/src/randomenv.h new file mode 100644 index 0000000000..46cea6f6f2 --- /dev/null +++ b/src/randomenv.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOMENV_H +#define BITCOIN_RANDOMENV_H + +#include + +/** Gather non-cryptographic environment data that changes over time. */ +void RandAddDynamicEnv(CSHA512& hasher); + +/** Gather non-cryptographic environment data that does not change over time. */ +void RandAddStaticEnv(CSHA512& hasher); + +#endif diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp new file mode 100644 index 0000000000..452a6b7202 --- /dev/null +++ b/src/test/random_tests.cpp @@ -0,0 +1,138 @@ +// Copyright (c) 2017-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/licenses/mit-license.php. + +#include + +// #include + +#include + +#include +#include + +extern bool g_mock_deterministic_tests; + +BOOST_AUTO_TEST_SUITE(random_tests) + +BOOST_AUTO_TEST_CASE(osrandom_tests) +{ + BOOST_CHECK(Random_SanityCheck()); +} + +BOOST_AUTO_TEST_CASE(fastrandom_tests) +{ + // Check that deterministic FastRandomContexts are deterministic + g_mock_deterministic_tests = true; + FastRandomContext ctx1(true); + FastRandomContext ctx2(true); + + for (int i = 10; i > 0; --i) { + BOOST_CHECK_EQUAL(GetRand(std::numeric_limits::max()), uint64_t{10393729187455219830U}); + BOOST_CHECK_EQUAL(GetRandInt(std::numeric_limits::max()), int{769702006}); + BOOST_CHECK_EQUAL(GetRandMicros(std::chrono::hours{1}).count(), 2917185654); + BOOST_CHECK_EQUAL(GetRandMillis(std::chrono::hours{1}).count(), 2144374); + } + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + BOOST_CHECK(ctx1.randbytes(17) == ctx2.randbytes(17)); + BOOST_CHECK(ctx1.rand256() == ctx2.rand256()); + BOOST_CHECK_EQUAL(ctx1.randbits(7), ctx2.randbits(7)); + BOOST_CHECK(ctx1.randbytes(128) == ctx2.randbytes(128)); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + BOOST_CHECK(ctx1.rand256() == ctx2.rand256()); + BOOST_CHECK(ctx1.randbytes(50) == ctx2.randbytes(50)); + + // Check that a nondeterministic ones are not + g_mock_deterministic_tests = false; + for (int i = 10; i > 0; --i) { + BOOST_CHECK(GetRand(std::numeric_limits::max()) != uint64_t{10393729187455219830U}); + BOOST_CHECK(GetRandInt(std::numeric_limits::max()) != int{769702006}); + BOOST_CHECK(GetRandMicros(std::chrono::hours{1}) != std::chrono::microseconds{2917185654}); + BOOST_CHECK(GetRandMillis(std::chrono::hours{1}) != std::chrono::milliseconds{2144374}); + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.rand256() != ctx4.rand256()); + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.randbytes(7) != ctx4.randbytes(7)); + } +} + +BOOST_AUTO_TEST_CASE(fastrandom_randbits) +{ + FastRandomContext ctx1; + FastRandomContext ctx2; + for (int bits = 0; bits < 63; ++bits) { + for (int j = 0; j < 1000; ++j) { + uint64_t rangebits = ctx1.randbits(bits); + BOOST_CHECK_EQUAL(rangebits >> bits, 0U); + uint64_t range = ((uint64_t)1) << bits | rangebits; + uint64_t rand = ctx2.randrange(range); + BOOST_CHECK(rand < range); + } + } +} + +/** Does-it-compile test for compatibility with standard C++11 RNG interface. */ +BOOST_AUTO_TEST_CASE(stdrandom_test) +{ + FastRandomContext ctx; + std::uniform_int_distribution distribution(3, 9); + for (int i = 0; i < 100; ++i) { + int x = distribution(ctx); + BOOST_CHECK(x >= 3); + BOOST_CHECK(x <= 9); + + std::vector test{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + std::shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } + Shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } + } +} + +/** Test that Shuffle reaches every permutation with equal probability. */ +BOOST_AUTO_TEST_CASE(shuffle_stat_test) +{ + FastRandomContext ctx(true); + uint32_t counts[5 * 5 * 5 * 5 * 5] = {0}; + for (int i = 0; i < 12000; ++i) { + int data[5] = {0, 1, 2, 3, 4}; + Shuffle(std::begin(data), std::end(data), ctx); + int pos = data[0] + data[1] * 5 + data[2] * 25 + data[3] * 125 + data[4] * 625; + ++counts[pos]; + } + unsigned int sum = 0; + double chi_score = 0.0; + for (int i = 0; i < 5 * 5 * 5 * 5 * 5; ++i) { + int i1 = i % 5, i2 = (i / 5) % 5, i3 = (i / 25) % 5, i4 = (i / 125) % 5, i5 = i / 625; + uint32_t count = counts[i]; + if (i1 == i2 || i1 == i3 || i1 == i4 || i1 == i5 || i2 == i3 || i2 == i4 || i2 == i5 || i3 == i4 || i3 == i5 || i4 == i5) { + BOOST_CHECK(count == 0); + } else { + chi_score += ((count - 100.0) * (count - 100.0)) / 100.0; + BOOST_CHECK(count > 50); + BOOST_CHECK(count < 150); + sum += count; + } + } + BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval + BOOST_CHECK(chi_score < 210.275); + BOOST_CHECK_EQUAL(sum, 12000U); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util.cpp b/src/util.cpp index a9a2907482..20acd3b00e 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include using namespace std; @@ -49,136 +48,6 @@ static std::map> dir_locks; /** Mutex to protect dir_locks. */ static std::mutex cs_dir_locks; -// Init OpenSSL library multithreading support -static CCriticalSection** ppmutexOpenSSL; -void locking_callback(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS -{ - if (mode & CRYPTO_LOCK) { - ENTER_CRITICAL_SECTION(*ppmutexOpenSSL[i]); - } else { - LEAVE_CRITICAL_SECTION(*ppmutexOpenSSL[i]); - } -} - -// Init -class CInit -{ -public: - CInit() - { - // Init OpenSSL library multithreading support - ppmutexOpenSSL = (CCriticalSection**)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(CCriticalSection*)); - for (int i = 0; i < CRYPTO_num_locks(); i++) - ppmutexOpenSSL[i] = new CCriticalSection(); - CRYPTO_set_locking_callback(locking_callback); - - // Check whether OpenSSL random number generator is properly seeded. If not, attempt to seed using RAND_poll(). - // Note that in versions of OpenSSL in the depends in Gridcoin (currently 1.1.1+), and modern Unix distros (using - // openSSL 1.1+ or modern Windows operating systems with the equivalent of /dev/urandom, the random number - // generator is automatically seeded on init, and periodically reseeded from trusted OS random sources. It is - // not necessary to manually reseed the RNG. Here we implement a check via RAND_status() to ensure the RNG - // is properly seeded. If not, we try 10 times (probably excessive) to seed the RNG via openSSL's entropy sources, - // breaking as soon as the return from RAND_poll() becomes 1 (successfully seeded). A 100 ms sleep is inserted - // between each try to ensure we give time for a short term unavailability of the OS entropy source to recover. - // If this falls through, we abort the application as we cannot have a non-functioning RNG. - bool seed_successful = RAND_status(); - if (!seed_successful) { - for (unsigned int i = 0; i < 10; ++i) - { - seed_successful = RAND_poll(); - - if (seed_successful) break; - - UninterruptibleSleep(std::chrono::milliseconds{100}); - } - - if (!seed_successful) { - tfm::format(std::cerr, "ERROR: %s: Unable to initialize the random number generator. Cannot continue. " - "Please check your operating system to ensure the random number source is " - "available. (This is /dev/urandom on Linux.)", - __func__); - std::abort(); - } - } - } - - ~CInit() - { - // Securely erase the memory used by the PRNG - RAND_cleanup(); - // Shutdown OpenSSL library multithreading support - CRYPTO_set_locking_callback(nullptr); - for (int i = 0; i < CRYPTO_num_locks(); i++) - delete ppmutexOpenSSL[i]; - OPENSSL_free(ppmutexOpenSSL); - } -} -instance_of_cinit; - -void RandAddSeed() -{ - // Seed with CPU performance counter - int64_t nCounter = GetPerformanceCounter(); - RAND_add(&nCounter, sizeof(nCounter), 1.5); - memory_cleanse(&nCounter, sizeof(nCounter)); -} - -void RandAddSeedPerfmon() -{ - RandAddSeed(); - - // Only need for OpenSSL < 1.1 and Win32, since 1.1 and greater seed properly from Windows, and on Linux seeds properly - // in all cases. -#if OPENSSL_VERSION_NUMBER < 0x10100000L && defined(WIN32) - // This can take up to 2 seconds, so only do it every 10 minutes - static int64_t nLastPerfmon; - if ( GetAdjustedTime() < nLastPerfmon + 10 * 60) - return; - nLastPerfmon = GetAdjustedTime(); - - // Don't need this on Linux, OpenSSL automatically uses /dev/urandom - // Seed with the entire set of perfmon data - unsigned char pdata[250000]; - memory_cleanse(pdata, sizeof(pdata)); - unsigned long nSize = sizeof(pdata); - long ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, pdata, &nSize); - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) - { - RAND_add(pdata, nSize, nSize/100.0); - memory_cleanse(pdata, nSize); - LogPrint(BCLog::LogFlags::NOISY, "rand", "RandAddSeed() %lu bytes", nSize); - } -#endif -} - -uint64_t GetRand(uint64_t nMax) -{ - if (nMax == 0) - return 0; - - // The range of the random source must be a multiple of the modulus - // to give every possible output value an equal possibility - uint64_t nRange = (std::numeric_limits::max() / nMax) * nMax; - uint64_t nRand = 0; - do - RAND_bytes((unsigned char*)&nRand, sizeof(nRand)); - while (nRand >= nRange); - return (nRand % nMax); -} - -int GetRandInt(int nMax) -{ - return GetRand(nMax); -} - -uint256 GetRandHash() -{ - uint256 hash; - RAND_bytes((unsigned char*)&hash, sizeof(hash)); - return hash; -} - int GetDayOfYear(int64_t timestamp) { try @@ -462,27 +331,6 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) #endif -uint32_t insecure_rand_Rz = 11; -uint32_t insecure_rand_Rw = 11; -void seed_insecure_rand(bool fDeterministic) -{ - //The seed values have some unlikely fixed points which we avoid. - if(fDeterministic) - { - insecure_rand_Rz = insecure_rand_Rw = 11; - } else { - uint32_t tmp; - do{ - RAND_bytes((unsigned char*)&tmp,4); - }while(tmp==0 || tmp==0x9068ffffU); - insecure_rand_Rz=tmp; - do{ - RAND_bytes((unsigned char*)&tmp,4); - }while(tmp==0 || tmp==0x464fffffU); - insecure_rand_Rw=tmp; - } -} - double Round(double d, int place) { const double accuracy = std::pow(10, place); From cf6bfce34eecd084b1de266b38ef3788acfcf19c Mon Sep 17 00:00:00 2001 From: div72 <60045611+div72@users.noreply.github.com> Date: Fri, 22 Oct 2021 18:02:24 +0300 Subject: [PATCH 3/4] random: update usages --- src/addrdb.cpp | 4 ++-- src/addrman.cpp | 1 + src/addrman.h | 5 +++-- src/gridcoin/scraper/scraper.cpp | 1 + src/gridcoin/staking/kernel.cpp | 1 + src/gridcoin/staking/spam.h | 1 + src/init.cpp | 17 ++++++++++++----- src/main.cpp | 9 ++++++--- src/miner.cpp | 6 ++---- src/net.cpp | 10 +++++++++- src/net.h | 1 - src/qt/winshutdownmonitor.cpp | 2 -- src/rpc/server.cpp | 3 ++- src/scheduler.cpp | 1 - src/script.cpp | 1 + src/test/dos_tests.cpp | 1 + src/test/merkle_tests.cpp | 1 + src/test/mruset_tests.cpp | 10 +++++----- src/test/scheduler_tests.cpp | 31 +++++++++++-------------------- src/test/test_gridcoin.cpp | 7 +++++++ src/test/util_tests.cpp | 31 ------------------------------- src/util.h | 28 ---------------------------- src/wallet/wallet.cpp | 21 ++++++++------------- 23 files changed, 74 insertions(+), 119 deletions(-) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 68d57a9f3c..c40d55a17c 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -9,7 +9,7 @@ #include #include #include -// #include +#include // #include #include // #include @@ -41,7 +41,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data { // Generate random temporary filename unsigned short randv = 0; - RAND_bytes((unsigned char*)&randv, sizeof(randv)); + GetRandBytes((unsigned char*)&randv, sizeof(randv)); std::string tmpfn = strprintf("%s.%04x", prefix, randv); // open temp output file, and associate with CAutoFile diff --git a/src/addrman.cpp b/src/addrman.cpp index 9602cbe21e..3fdd12cd2c 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -3,6 +3,7 @@ // file COPYING or https://opensource.org/licenses/mit-license.php. #include "addrman.h" +#include "random.h" #include "streams.h" using namespace std; diff --git a/src/addrman.h b/src/addrman.h index 92764b72c1..60eab366ff 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -7,6 +7,7 @@ #include "netbase.h" #include "protocol.h" +#include "random.h" #include "util.h" #include "sync.h" @@ -383,7 +384,7 @@ class CAddrMan CAddrMan() : vRandom(0), vvTried(ADDRMAN_TRIED_BUCKET_COUNT, std::vector(0)), vvNew(ADDRMAN_NEW_BUCKET_COUNT, std::set()) { nKey.resize(32); - RAND_bytes(&nKey[0], 32); + GetRandBytes(nKey.data(), 32); nIdCount = 0; nTried = 0; @@ -504,7 +505,7 @@ class CAddrMan { LOCK(cs); std::vector().swap(vRandom); - RAND_bytes(&nKey[0], 32); + GetRandBytes(nKey.data(), 32); vvTried = std::vector>(ADDRMAN_TRIED_BUCKET_COUNT, std::vector(0)); vvNew = std::vector>(ADDRMAN_NEW_BUCKET_COUNT, std::set()); // Will need for Bitcoin rebase diff --git a/src/gridcoin/scraper/scraper.cpp b/src/gridcoin/scraper/scraper.cpp index 50e9034b36..7e1162c92a 100755 --- a/src/gridcoin/scraper/scraper.cpp +++ b/src/gridcoin/scraper/scraper.cpp @@ -4,6 +4,7 @@ #include "main.h" #include "node/ui_interface.h" +#include "random.h" #include "gridcoin/appcache.h" #include "gridcoin/beacon.h" diff --git a/src/gridcoin/staking/kernel.cpp b/src/gridcoin/staking/kernel.cpp index 3e62b665db..b8c4503eff 100644 --- a/src/gridcoin/staking/kernel.cpp +++ b/src/gridcoin/staking/kernel.cpp @@ -8,6 +8,7 @@ #include "gridcoin/staking/kernel.h" #include "txdb.h" #include "main.h" +#include "random.h" #include "streams.h" #include "util.h" diff --git a/src/gridcoin/staking/spam.h b/src/gridcoin/staking/spam.h index 89fa392573..41a27eaa1a 100644 --- a/src/gridcoin/staking/spam.h +++ b/src/gridcoin/staking/spam.h @@ -7,6 +7,7 @@ #include "kernel.h" #include "uint256.h" +#include "random.h" #include #include diff --git a/src/init.cpp b/src/init.cpp index b6bc0cc267..c59fa6fc01 100755 --- a/src/init.cpp +++ b/src/init.cpp @@ -11,6 +11,7 @@ #include "txdb.h" #include "wallet/walletdb.h" #include "banman.h" +#include "random.h" #include "rpc/server.h" #include "init.h" #include "node/ui_interface.h" @@ -606,6 +607,10 @@ bool InitSanityCheck(void) return false; } + if (!Random_SanityCheck()) { + return InitError("OS cryptographic RNG sanity check failure. Aborting."); + } + return true; } @@ -929,6 +934,7 @@ bool AppInit2(ThreadHandlerPtr threads) // Initialize internal hashing code with SSE/AVX2 optimizations. In the future we will also have ARM/NEON optimizations. std::string sha256_algo = SHA256AutoDetect(); LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); + RandomInit(); LogPrintf("Block version 11 hard fork configured for block %d", Params().GetConsensus().BlockV11Height); @@ -1268,12 +1274,10 @@ bool AppInit2(ThreadHandlerPtr threads) if (fFirstRun) { - // Create new keyUser and set as default key - RandAddSeedPerfmon(); - // So Clang doesn't complain, even though we are really essentially single-threaded here. LOCK(pwalletMain->cs_wallet); + // Create new keyUser and set as default key CPubKey newDefaultKey; if (pwalletMain->GetKeyFromPool(newDefaultKey, false)) { pwalletMain->SetDefaultKey(newDefaultKey); @@ -1385,8 +1389,6 @@ bool AppInit2(ThreadHandlerPtr threads) if (!CheckDiskSpace()) return false; - RandAddSeedPerfmon(); - if (!GRC::Initialize(threads, pindexBest)) { return false; } @@ -1426,6 +1428,11 @@ bool AppInit2(ThreadHandlerPtr threads) CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(std::bind(&TraceThread, "scheduler", serviceLoop)); + // Gather some entropy once per minute. + scheduler.scheduleEvery([]{ + RandAddPeriodic(); + }, std::chrono::minutes{1}); + // TODO: Do we need this? It would require porting the Bitcoin signal handler. // GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); diff --git a/src/main.cpp b/src/main.cpp index aa5a3c944d..eceb5a3604 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -34,6 +34,7 @@ #include "node/blockstorage.h" #include "policy/fees.h" #include "policy/policy.h" +#include "random.h" #include "validation.h" #include @@ -2918,8 +2919,6 @@ bool static AlreadyHave(CTxDB& txdb, const CInv& inv) bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived) { - RandAddSeedPerfmon(); - LogPrint(BCLog::LogFlags::NOISY, "received: %s from %s (%" PRIszu " bytes)", strCommand, pfrom->addrName, vRecv.size()); if (strCommand == "aries") @@ -3782,6 +3781,10 @@ bool ProcessMessages(CNode* pfrom) // Checksum CDataStream& vRecv = msg.vRecv; uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize); + + // We just received a message off the wire, harvest entropy from the time (and the message checksum) + RandAddEvent(ReadLE32(hash.begin())); + // TODO: hardcoded checksum size; // will no longer be used once we adopt CNetMessage from Bitcoin uint8_t nChecksum[CMessageHeader::CHECKSUM_SIZE]; @@ -3878,7 +3881,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle) { uint64_t nonce = 0; while (nonce == 0) { - RAND_bytes((unsigned char*)&nonce, sizeof(nonce)); + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); } pto->fPingQueued = false; pto->nPingUsecStart = GetTimeMicros(); diff --git a/src/miner.cpp b/src/miner.cpp index 2eaecd1ae3..450d7139a4 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -21,6 +21,7 @@ #include "gridcoin/tally.h" #include "policy/policy.h" #include "policy/fees.h" +#include "random.h" #include "util.h" #include "validation.h" #include "wallet/wallet.h" @@ -28,7 +29,6 @@ #include #include #include -#include using namespace std; @@ -685,9 +685,7 @@ void SplitCoinStakeOutput(CBlock &blocknew, int64_t &nReward, bool &fEnableStake // the percentages is done. if (vSideStakeAlloc.size() > nMaxSideStakeOutputs) { - unsigned int seed = static_cast(GetAdjustedTime()); - - std::shuffle(vSideStakeAlloc.begin(), vSideStakeAlloc.end(), std::default_random_engine(seed)); + Shuffle(vSideStakeAlloc.begin(), vSideStakeAlloc.end(), FastRandomContext()); } // Initialize remaining stake output value to the total value of output for stake, which also includes diff --git a/src/net.cpp b/src/net.cpp index 4bfe5d2a08..f12a1de51f 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -12,6 +12,7 @@ #include "net.h" #include "init.h" #include "node/ui_interface.h" +#include "random.h" #include "util.h" #include "util/threadnames.h" @@ -405,6 +406,10 @@ CNode* ConnectNode(CAddress addrConnect, const char *pszDest) } pnode->nTimeConnected = GetAdjustedTime(); + + // We're making a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)pnode->GetId()); + return pnode; } else @@ -481,7 +486,7 @@ void CNode::PushVersion() int64_t nTime = GetAdjustedTime(); CAddress addrYou = (addr.IsRoutable() && !IsProxy(addr) ? addr : CAddress(CService("0.0.0.0",0))); CAddress addrMe = GetLocalAddress(&addr); - RAND_bytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce)); + GetRandBytes((unsigned char*)&nLocalHostNonce, sizeof(nLocalHostNonce)); LogPrint(BCLog::LogFlags::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%s", PROTOCOL_VERSION, nBestHeight, addrMe.ToString(), addrYou.ToString(), addr.ToString()); @@ -966,6 +971,9 @@ void ThreadSocketHandler2(void* parg) LOCK(cs_vNodes); vNodes.push_back(pnode); } + + // We received a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)pnode->GetId()); } } diff --git a/src/net.h b/src/net.h index 09d3f95f9f..123276f62e 100644 --- a/src/net.h +++ b/src/net.h @@ -9,7 +9,6 @@ #include #include #include -#include #include "netbase.h" #include "mruset.h" diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 20b2d3c597..bcbcef6cc5 100755 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -12,8 +12,6 @@ #include -#include - // If we don't want a message to be processed by Qt, return true and set result to // the value that the window procedure should return. Otherwise return false. bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 81cb936d35..c9c2115458 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -11,6 +11,7 @@ #include "server.h" #include "client.h" #include "protocol.h" +#include "random.h" #include "wallet/db.h" #include "util.h" @@ -583,7 +584,7 @@ void StartRPCThreads() (gArgs.GetArg("-rpcuser", "") == gArgs.GetArg("-rpcpassword", "")))) { unsigned char rand_pwd[32]; - RAND_bytes(rand_pwd, 32); + GetRandBytes(rand_pwd, sizeof(rand_pwd)); string strWhatAmI = "To use gridcoind"; if (gArgs.IsArgSet("-server")) strWhatAmI = strprintf(_("To use the %s option"), "\"-server\""); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 4c80423274..b9b0cbe8f0 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,7 +4,6 @@ #include -//#include #include #include diff --git a/src/script.cpp b/src/script.cpp index e9e563f3d7..266a157c89 100644 --- a/src/script.cpp +++ b/src/script.cpp @@ -10,6 +10,7 @@ using namespace std; #include "bignum.h" #include "key.h" #include "main.h" +#include "random.h" #include "streams.h" #include "sync.h" #include "util.h" diff --git a/src/test/dos_tests.cpp b/src/test/dos_tests.cpp index 592b843620..1397c74ccc 100755 --- a/src/test/dos_tests.cpp +++ b/src/test/dos_tests.cpp @@ -10,6 +10,7 @@ #include "wallet/wallet.h" #include "net.h" #include "util.h" +#include "random.h" #include "banman.h" #include diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 6b51e63bf6..bcc6855186 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or https://opensource.org/licenses/mit-license.php. #include +#include "random.h" #include diff --git a/src/test/mruset_tests.cpp b/src/test/mruset_tests.cpp index 64a6678a73..b77a7d9427 100644 --- a/src/test/mruset_tests.cpp +++ b/src/test/mruset_tests.cpp @@ -3,7 +3,7 @@ using namespace std; #include "mruset.h" -#include "util.h" +#include "random.h" #define NUM_TESTS 16 #define MAX_SIZE 100 @@ -31,12 +31,12 @@ BOOST_AUTO_TEST_SUITE(mruset_tests) // Test that an mruset behaves like a set, as long as no more than MAX_SIZE elements are in it BOOST_AUTO_TEST_CASE(mruset_like_set) { - + FastRandomContext ctx; for (int nTest=0; nTest mru(MAX_SIZE); for (int nAction=0; nAction<3*MAX_SIZE; nAction++) { - int n = GetRandInt(2 * MAX_SIZE); - mru.insert(n); + mru.insert(ctx.randrange(2 * MAX_SIZE)); BOOST_CHECK(mru.size() <= MAX_SIZE); } } diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 8f6ffeec66..ad44fe734f 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -2,10 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -// Reserved for when we port Bitcoin's random.h -//#include +#include #include -#include +// #include #include #include @@ -46,18 +45,10 @@ BOOST_AUTO_TEST_CASE(manythreads) std::mutex counterMutex[10]; int counter[10] = { 0 }; - - // We can't use this until we port Bitcoin's random.h, so we will go back and use the older non-deterministic - // random number generator. - /* - FastRandomContext rng{true}; + FastRandomContext rng{/* fDeterministic */ true}; auto zeroToNine = [](FastRandomContext& rc) -> int { return rc.randrange(10); }; // [0, 9] auto randomMsec = [](FastRandomContext& rc) -> int { return -11 + (int)rc.randrange(1012); }; // [-11, 1000] auto randomDelta = [](FastRandomContext& rc) -> int { return -1000 + (int)rc.randrange(2001); }; // [-1000, 1000] - */ - auto zeroToNine = []() { return GetRandInt(10); }; // [0,9] - auto randomMsec = []() { return GetRandInt(1012) - 11; }; // [-11, 1000] - auto randomDelta = []() { return GetRandInt(2001) - 1000; }; // [-1000, 1000] std::chrono::system_clock::time_point start = std::chrono::system_clock::now(); std::chrono::system_clock::time_point now = start; @@ -66,12 +57,12 @@ BOOST_AUTO_TEST_CASE(manythreads) BOOST_CHECK(nTasks == 0); for (int i = 0; i < 100; ++i) { - std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec()); - std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec()); - int whichCounter = zeroToNine(); + std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); + std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); CScheduler::Function f = std::bind(µTask, std::ref(microTasks), std::ref(counterMutex[whichCounter]), std::ref(counter[whichCounter]), - randomDelta(), tReschedule); + randomDelta(rng), tReschedule); microTasks.schedule(f, t); } nTasks = microTasks.getQueueInfo(first, last); @@ -91,12 +82,12 @@ BOOST_AUTO_TEST_CASE(manythreads) for (int i = 0; i < 5; i++) microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); for (int i = 0; i < 100; i++) { - std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec()); - std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec()); - int whichCounter = zeroToNine(); + std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); + std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); + int whichCounter = zeroToNine(rng); CScheduler::Function f = std::bind(µTask, std::ref(microTasks), std::ref(counterMutex[whichCounter]), std::ref(counter[whichCounter]), - randomDelta(), tReschedule); + randomDelta(rng), tReschedule); microTasks.schedule(f, t); } diff --git a/src/test/test_gridcoin.cpp b/src/test/test_gridcoin.cpp index c2002da353..802145afc1 100644 --- a/src/test/test_gridcoin.cpp +++ b/src/test/test_gridcoin.cpp @@ -9,11 +9,16 @@ #include "dbwrapper.h" #include "wallet/db.h" #include "main.h" +#include "random.h" #include "wallet/wallet.h" extern CWallet* pwalletMain; extern leveldb::DB *txdb; extern CClientUIInterface uiInterface; +/** + * Flag to make GetRand in random.h return the same number + */ +extern bool g_mock_deterministic_tests; extern void noui_connect(); extern leveldb::Options GetOptions(); @@ -50,6 +55,7 @@ struct TestingSetup { assert(!g_banman); // Create ban manager instance. g_banman = std::make_unique(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + g_mock_deterministic_tests = true; } ~TestingSetup() { @@ -59,6 +65,7 @@ struct TestingSetup { g_banman.reset(); delete txdb; txdb = nullptr; + g_mock_deterministic_tests = false; } }; diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index ed27d20c60..297e4af1a6 100755 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -350,37 +350,6 @@ BOOST_AUTO_TEST_CASE(util_IsHexNumber) } -BOOST_AUTO_TEST_CASE(util_seed_insecure_rand) -{ - int i; - int count=0; - - seed_insecure_rand(true); - - - for (int mod=2;mod<11;mod++) - { - int mask = 1; - // Really rough binomial confidence approximation. - int err = 30*10000./mod*sqrt((1./mod*(1-1./mod))/10000.); - //mask is 2^ceil(log2(mod))-1 - while(mask=(uint32_t)mod); - count += rval==0; - } - BOOST_CHECK(count<=10000/mod+err); - BOOST_CHECK(count>=10000/mod-err); - } -} - BOOST_AUTO_TEST_CASE(util_TimingResistantEqual) { BOOST_CHECK(TimingResistantEqual(std::string(""), std::string(""))); diff --git a/src/util.h b/src/util.h index d3a22ed3c1..bbab5af554 100644 --- a/src/util.h +++ b/src/util.h @@ -88,9 +88,6 @@ extern bool fLogTimestamps; extern bool fReopenDebugLog; extern bool fDevbuildCripple; -void RandAddSeed(); -void RandAddSeedPerfmon(); - void LogException(std::exception* pex, const char* pszThread); void PrintException(std::exception* pex, const char* pszThread); void PrintExceptionContinue(std::exception* pex, const char* pszThread); @@ -119,9 +116,6 @@ bool TryCreateDirectories(const fs::path& p); //! std::string GetFileContents(const fs::path filepath); -int GetRandInt(int nMax); -uint64_t GetRand(uint64_t nMax); -uint256 GetRandHash(); int64_t GetTimeOffset(); int64_t GetAdjustedTime(); std::string FormatFullVersion(); @@ -194,28 +188,6 @@ inline int64_t GetPerformanceCounter() return nCounter; } -/** - * MWC RNG of George Marsaglia - * This is intended to be fast. It has a period of 2^59.3, though the - * least significant 16 bits only have a period of about 2^30.1. - * - * @return random value - */ -extern uint32_t insecure_rand_Rz; -extern uint32_t insecure_rand_Rw; -static inline uint32_t insecure_rand(void) -{ - insecure_rand_Rz=36969*(insecure_rand_Rz&65535)+(insecure_rand_Rz>>16); - insecure_rand_Rw=18000*(insecure_rand_Rw&65535)+(insecure_rand_Rw>>16); - return (insecure_rand_Rw<<16)+insecure_rand_Rz; -} - -/** - * Seed insecure_rand using the random pool. - * @param Deterministic Use a deterministic seed - */ -void seed_insecure_rand(bool fDeterministic=false); - /** Median filter over a stream of values. * Returns the median of the last N numbers */ diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 8bf58f8b48..8c47a52e50 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -13,6 +13,7 @@ #include "wallet/coincontrol.h" #include #include +#include "random.h" #include "rpc/server.h" #include "rpc/client.h" #include "rpc/protocol.h" @@ -20,12 +21,12 @@ #include "main.h" #include "util.h" #include -#include #include "gridcoin/staking/kernel.h" #include "gridcoin/support/block_finder.h" #include "policy/fees.h" #include "node/blockstorage.h" + using namespace std; extern bool fQtActive; @@ -91,7 +92,6 @@ CPubKey CWallet::GenerateNewKey() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) AssertLockHeld(cs_wallet); // mapKeyMetadata bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets - RandAddSeedPerfmon(); CKey key; key.MakeNewKey(fCompressed); @@ -310,16 +310,14 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) return false; CKeyingMaterial vMasterKey; - RandAddSeedPerfmon(); vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - RAND_bytes(&vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); + GetStrongRandBytes(vMasterKey.data(), WALLET_CRYPTO_KEY_SIZE); CMasterKey kMasterKey(nDerivationMethodIndex); - RandAddSeedPerfmon(); kMasterKey.vchSalt.resize(WALLET_CRYPTO_SALT_SIZE); - RAND_bytes(&kMasterKey.vchSalt[0], WALLET_CRYPTO_SALT_SIZE); + GetStrongRandBytes(kMasterKey.vchSalt.data(), WALLET_CRYPTO_SALT_SIZE); CCrypter crypter; int64_t nStartTime = GetTimeMillis(); @@ -1416,7 +1414,7 @@ static void ApproximateBestSubset(vector > > vValue; int64_t nTotalLower = 0; - auto seed = static_cast(GetTimeMicros()); - std::shuffle(vCoins.begin(), vCoins.end(), std::default_random_engine(seed)); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); for (auto output : vCoins) { @@ -1764,9 +1761,7 @@ bool CWallet::SelectCoinsForStaking(unsigned int nSpendTime, std::vector(GetAdjustedTime()); - - std::shuffle(vCoinsRet.begin(), vCoinsRet.end(), std::default_random_engine(seed)); + Shuffle(vCoinsRet.begin(), vCoinsRet.end(), FastRandomContext()); } g_timer.GetTimes(function + "shuffle", "miner"); From 73ead7df934a619f4f797b05386f2d3eab64469f Mon Sep 17 00:00:00 2001 From: div72 <60045611+div72@users.noreply.github.com> Date: Fri, 22 Oct 2021 14:48:42 +0300 Subject: [PATCH 4/4] devtools: fix deterministic test coverage check script --- .../devtools/test_deterministic_coverage.sh | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index d377da748f..9dc63fdbea 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -13,28 +13,9 @@ export LC_ALL=C GCOV_EXECUTABLE="gcov" # Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. -NON_DETERMINISTIC_TESTS=( - "blockfilter_index_tests/blockfilter_index_initial_sync" # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... } - "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) - "fs_tests/fsbridge_fstream" # deterministic test failure? - "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() - "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() - "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) -) +NON_DETERMINISTIC_TESTS=() -TEST_BITCOIN_BINARY="src/test/test_bitcoin" +TEST_BITCOIN_BINARY="src/test/test_gridcoin" print_usage() { echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]"