Merge pull request #4355 from kittywhiskers/bfilters

merge bitcoin#16787, bitcoin#17474, bitcoin#17687, bitcoin#18165, bitcoin#18877, bitcoin#18960, bitcoin#19010, bitcoin#19044, bitcoin#19070: block filters

src/blockfilter.cpp

@@ -4,6 +4,7 @@
 
 #include <mutex>
 #include <sstream>
+#include <set>
 
 #include <blockfilter.h>
 #include <crypto/siphash.h>
@@ -221,15 +222,14 @@ bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type
     return false;
 }
 
-const std::vector<BlockFilterType>& AllBlockFilterTypes()
+const std::set<BlockFilterType>& AllBlockFilterTypes()
 {
-    static std::vector<BlockFilterType> types;
+    static std::set<BlockFilterType> types;
 
     static std::once_flag flag;
     std::call_once(flag, []() {
-            types.reserve(g_filter_types.size());
             for (auto entry : g_filter_types) {
-                types.push_back(entry.first);
+                types.insert(entry.first);
             }
         });
 

src/blockfilter.h

@@ -7,6 +7,7 @@
 
 #include <stdint.h>
 #include <string>
+#include <set>
 #include <unordered_set>
 #include <vector>
 
@@ -96,7 +97,7 @@ const std::string& BlockFilterTypeName(BlockFilterType filter_type);
 bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type);
 
 /** Get a list of known filter types. */
-const std::vector<BlockFilterType>& AllBlockFilterTypes();
+const std::set<BlockFilterType>& AllBlockFilterTypes();
 
 /** Get a comma-separated list of known filter type names. */
 const std::string& ListBlockFilterTypes();
@@ -142,8 +143,8 @@ class BlockFilter
 
     template <typename Stream>
     void Serialize(Stream& s) const {
-        s << m_block_hash
-          << static_cast<uint8_t>(m_filter_type)
+        s << static_cast<uint8_t>(m_filter_type)
+          << m_block_hash
           << m_filter.GetEncoded();
     }
 
@@ -152,8 +153,8 @@ class BlockFilter
         std::vector<unsigned char> encoded_filter;
         uint8_t filter_type;
 
-        s >> m_block_hash
-          >> filter_type
+        s >> filter_type
+          >> m_block_hash
           >> encoded_filter;
 
         m_filter_type = static_cast<BlockFilterType>(filter_type);

src/index/blockfilterindex.cpp

@@ -32,6 +32,12 @@ constexpr char DB_FILTER_POS = 'P';
 constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
 /** The pre-allocation chunk size for fltr?????.dat files */
 constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
+/** Maximum size of the cfheaders cache
+ *  We have a limit to prevent a bug in filling this cache
+ *  potentially turning into an OOM. At 2000 entries, this cache
+ *  is big enough for a 2,000,000 length block chain, which
+ *  we should be enough until ~2047. */
+constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000};
 
 namespace {
 
@@ -384,13 +390,32 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter&
     return ReadFilterFromDisk(entry.pos, filter_out);
 }
 
-bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const
+bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out)
 {
+    LOCK(m_cs_headers_cache);
+
+    bool is_checkpoint{block_index->nHeight % CFCHECKPT_INTERVAL == 0};
+
+    if (is_checkpoint) {
+        // Try to find the block in the headers cache if this is a checkpoint height.
+        auto header = m_headers_cache.find(block_index->GetBlockHash());
+        if (header != m_headers_cache.end()) {
+            header_out = header->second;
+            return true;
+        }
+    }
+
     DBVal entry;
     if (!LookupOne(*m_db, block_index, entry)) {
         return false;
     }
 
+    if (is_checkpoint &&
+        m_headers_cache.size() < CF_HEADERS_CACHE_MAX_SZ) {
+        // Add to the headers cache if this is a checkpoint height.
+        m_headers_cache.emplace(block_index->GetBlockHash(), entry.header);
+    }
+
     header_out = entry.header;
     return true;
 }

src/index/blockfilterindex.h

@@ -10,6 +10,14 @@
 #include <flatfile.h>
 #include <index/base.h>
 
+/** Interval between compact filter checkpoints. See BIP 157. */
+static constexpr int CFCHECKPT_INTERVAL = 1000;
+
+struct FilterHeaderHasher
+{
+    size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); }
+};
+
 /**
  * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of
  * blocks by height. An index is constructed for each supported filter type with its own database
@@ -30,6 +38,10 @@ class BlockFilterIndex final : public BaseIndex
     bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const;
     size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
 
+    Mutex m_cs_headers_cache;
+    /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */
+    std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache);
+
 protected:
     bool Init() override;
 
@@ -54,7 +66,7 @@ class BlockFilterIndex final : public BaseIndex
     bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const;
 
     /** Get a single filter header by block. */
-    bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const;
+    bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out);
 
     /** Get a range of filters between two heights on a chain. */
     bool LookupFilterRange(int start_height, const CBlockIndex* stop_index,

src/init.cpp

@@ -82,6 +82,7 @@
 
 #include <stdint.h>
 #include <stdio.h>
+#include <set>
 
 #include <bls/bls.h>
 
@@ -552,6 +553,7 @@ void SetupServerArgs()
     gArgs.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h), 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -noonion to disable (default: -proxy)", false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", false, OptionsCategory::CONNECTION);
+    gArgs.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), false, OptionsCategory::CONNECTION);
@@ -1210,7 +1212,7 @@ int nUserMaxConnections;
 int nFD;
 ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED);
 int64_t peer_connect_timeout;
-std::vector<BlockFilterType> g_enabled_filter_types;
+std::set<BlockFilterType> g_enabled_filter_types;
 
 } // namespace
 
@@ -1294,16 +1296,24 @@ bool AppInitParameterInteraction()
         g_enabled_filter_types = AllBlockFilterTypes();
     } else if (blockfilterindex_value != "0") {
         const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex");
-        g_enabled_filter_types.reserve(names.size());
         for (const auto& name : names) {
             BlockFilterType filter_type;
             if (!BlockFilterTypeByName(name, filter_type)) {
                 return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name));
             }
-            g_enabled_filter_types.push_back(filter_type);
+            g_enabled_filter_types.insert(filter_type);
         }
     }
 
+    // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled.
+    if (gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)) {
+        if (g_enabled_filter_types.count(BlockFilterType::BASIC_FILTER) != 1) {
+            return InitError(_("Cannot set -peerblockfilters without -blockfilterindex."));
+        }
+
+        nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS);
+    }
+
     // if using block pruning, then disallow txindex and require disabling governance validation
     if (gArgs.GetArg("-prune", 0)) {
         if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))